build: remove support for building without X10 (#1000)

compnerd · web-flow · commit cd1b7bcd2bdd · 2020-06-12T07:46:33.000-07:00
X10 is now supports all the targets.  It was part of the 0.9 release.
Remove the non-X10 paths as that is no longer really tested.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -22,94 +22,75 @@ set(CMAKE_CXX_STANDARD 14)
 set(CMAKE_CXX_STANDARD_REQUIRED TRUE)
 set(CMAKE_Swift_MODULE_DIRECTORY ${CMAKE_BINARY_DIR}/swift)
 
-option(BUILD_X10 "enable the x10 tensor library" OFF)
 option(USE_BUNDLED_X10
   "Use the x10 library bundled in the active Swift toolchain" OFF)
 option(USE_BUNDLED_CTENSORFLOW
   "Use the CTensorFlow module bundled in the active Swift toolchain" OFF)
 
-if(BUILD_X10)
-  find_package(X10)
-  if(NOT X10_FOUND AND NOT USE_BUNDLED_X10)
-    include(ExternalProject)
+find_package(X10)
+if(NOT X10_FOUND AND NOT USE_BUNDLED_X10)
+  include(ExternalProject)
 
-    if(CMAKE_SYSTEM_NAME STREQUAL Windows)
-      set(VISIBILITY_FLAGS "")
-    else()
-      set(VISIBILITY_FLAGS --cxxopt=-fvisibility=hidden --cxxopt=-fvisibility-inlines-hidden)
-    endif()
+  if(CMAKE_SYSTEM_NAME STREQUAL Windows)
+    set(VISIBILITY_FLAGS "")
+  else()
+    set(VISIBILITY_FLAGS --cxxopt=-fvisibility=hidden --cxxopt=-fvisibility-inlines-hidden)
+  endif()
 
-    ExternalProject_Add(libtensorflow
-      GIT_REPOSITORY
-        git://github.com/tensorflow/tensorflow
-      GIT_TAG
-        r2.2
-      UPDATE_DISCONNECTED
-        TRUE
-      CONFIGURE_COMMAND
-        COMMAND
-          ${CMAKE_COMMAND} -E create_symlink ${PROJECT_SOURCE_DIR}/Sources/CX10 <SOURCE_DIR>/swift_bindings
-        COMMAND
-          ${CMAKE_COMMAND} -E create_symlink ${PROJECT_SOURCE_DIR}/Sources/x10/xla_client <SOURCE_DIR>/tensorflow/compiler/xla/xla_client
-        COMMAND
-          ${CMAKE_COMMAND} -E create_symlink ${PROJECT_SOURCE_DIR}/Sources/x10/xla_tensor <SOURCE_DIR>/tensorflow/compiler/tf2xla/xla_tensor
-        COMMAND
-          # Note that the configure script is affected by the following environment variables:
-          #   TF_CUDA_VERSION
-          #   TF_NEED_CUDA
-          #   TF_CUDNN_VERSION
-          #   CUDA_TOOLKIT_PATH
-          #   CUDNN_INSTALL_PATH
-          sh -c "yes '' | <SOURCE_DIR>/configure"
-      BUILD_COMMAND
-        COMMAND
-          rm -rf <SOURCE_DIR>/bazel-bin # ${CMAKE_COMMAND} -E rm -Rrf <SOURCE_DIR>/bazel-bin
-        COMMAND
-          bazel build ${VISIBILITY_FLAGS} -c opt --define framework_shared_object=false //tensorflow/compiler/tf2xla/xla_tensor:x10 --nocheck_visibility
-        COMMAND
-          bazel shutdown
-      INSTALL_COMMAND
-        "" # we do not do any installation, we fetch out the artifacts manually
-      BUILD_IN_SOURCE
-        TRUE
-      BUILD_BYPRODUCTS
-        <SOURCE_DIR>/bazel-bin/tensorflow/compiler/tf2xla/xla_tensor/${CMAKE_SHARED_LIBRARY_PREFIX}x10${CMAKE_SHARED_LIBRARY_SUFFIX}
-      USES_TERMINAL_BUILD
-        TRUE
-      STEP_TARGETS
-        build)
-    ExternalProject_Get_Property(libtensorflow SOURCE_DIR)
+  ExternalProject_Add(libtensorflow
+    GIT_REPOSITORY
+      git://github.com/tensorflow/tensorflow
+    GIT_TAG
+      r2.2
+    UPDATE_DISCONNECTED
+      TRUE
+    CONFIGURE_COMMAND
+      COMMAND
+        ${CMAKE_COMMAND} -E create_symlink ${PROJECT_SOURCE_DIR}/Sources/CX10 <SOURCE_DIR>/swift_bindings
+      COMMAND
+        ${CMAKE_COMMAND} -E create_symlink ${PROJECT_SOURCE_DIR}/Sources/x10/xla_client <SOURCE_DIR>/tensorflow/compiler/xla/xla_client
+      COMMAND
+        ${CMAKE_COMMAND} -E create_symlink ${PROJECT_SOURCE_DIR}/Sources/x10/xla_tensor <SOURCE_DIR>/tensorflow/compiler/tf2xla/xla_tensor
+      COMMAND
+        # Note that the configure script is affected by the following environment variables:
+        #   TF_CUDA_VERSION
+        #   TF_NEED_CUDA
+        #   TF_CUDNN_VERSION
+        #   CUDA_TOOLKIT_PATH
+        #   CUDNN_INSTALL_PATH
+        sh -c "yes '' | <SOURCE_DIR>/configure"
+    BUILD_COMMAND
+      COMMAND
+        rm -rf <SOURCE_DIR>/bazel-bin # ${CMAKE_COMMAND} -E rm -Rrf <SOURCE_DIR>/bazel-bin
+      COMMAND
+        bazel build ${VISIBILITY_FLAGS} -c opt --define framework_shared_object=false //tensorflow/compiler/tf2xla/xla_tensor:x10 --nocheck_visibility
+      COMMAND
+        bazel shutdown
+    INSTALL_COMMAND
+      "" # we do not do any installation, we fetch out the artifacts manually
+    BUILD_IN_SOURCE
+      TRUE
+    BUILD_BYPRODUCTS
+      <SOURCE_DIR>/bazel-bin/tensorflow/compiler/tf2xla/xla_tensor/${CMAKE_SHARED_LIBRARY_PREFIX}x10${CMAKE_SHARED_LIBRARY_SUFFIX}
+    USES_TERMINAL_BUILD
+      TRUE
+    STEP_TARGETS
+      build)
+  ExternalProject_Get_Property(libtensorflow SOURCE_DIR)
 
-    set(X10_LIBRARY ${SOURCE_DIR}/bazel-bin/tensorflow/compiler/tf2xla/xla_tensor/${CMAKE_SHARED_LIBRARY_PREFIX}x10${CMAKE_SHARED_LIBRARY_SUFFIX})
-    set(X10_INCLUDE_DIRS "${SOURCE_DIR};${SOURCE_DIR}/bazel-bin;${SOURCE_DIR}/bazel-libtensorflow/external/com_google_absl;${SOURCE_DIR}/bazel-libtensorflow/external/com_google_protobuf/src;${SOURCE_DIR}/bazel-libtensorflow/external/eigen_archive")
-    add_library(x10 IMPORTED UNKNOWN)
-    set_target_properties(x10 PROPERTIES
-      IMPORTED_LOCATION ${X10_LIBRARY}
-      INTERFACE_INCLUDE_DIRECTORIES "${X10_INCLUDE_DIRS}")
-    add_dependencies(x10
-      libtensorflow-build)
+  set(X10_LIBRARY ${SOURCE_DIR}/bazel-bin/tensorflow/compiler/tf2xla/xla_tensor/${CMAKE_SHARED_LIBRARY_PREFIX}x10${CMAKE_SHARED_LIBRARY_SUFFIX})
+  set(X10_INCLUDE_DIRS "${SOURCE_DIR};${SOURCE_DIR}/bazel-bin;${SOURCE_DIR}/bazel-libtensorflow/external/com_google_absl;${SOURCE_DIR}/bazel-libtensorflow/external/com_google_protobuf/src;${SOURCE_DIR}/bazel-libtensorflow/external/eigen_archive")
+  add_library(x10 IMPORTED UNKNOWN)
+  set_target_properties(x10 PROPERTIES
+    IMPORTED_LOCATION ${X10_LIBRARY}
+    INTERFACE_INCLUDE_DIRECTORIES "${X10_INCLUDE_DIRS}")
+  add_dependencies(x10
+    libtensorflow-build)
 
-    get_target_property(DIRECTORIES x10 INTERFACE_INCLUDE_DIRECTORIES)
-    foreach(directory ${DIRECTORIES})
-      file(MAKE_DIRECTORY ${directory})
-    endforeach()
-  endif()
-endif()
-
-if(BUILD_X10)
-  if(NOT X10_FOUND AND NOT USE_BUNDLED_X10)
-    add_library(tensorflow UNKNOWN IMPORTED)
-    set_target_properties(tensorflow PROPERTIES
-      IMPORTED_LOCATION ${X10_LIBRARY}
-      INTERFACE_INCLUDE_DIRECTORIES "${SOURCE_DIR}")
-  else()
-    add_library(tensorflow UNKNOWN IMPORTED)
-    set_target_properties(tensorflow PROPERTIES
-      IMPORTED_LOCATION ${X10_LIBRARY}
-      INTERFACE_INCLUDE_DIRECTORIES "${X10_INCLUDE_DIRS}")
-  endif()
-else()
-  find_package(TensorFlow REQUIRED)
+  get_target_property(DIRECTORIES x10 INTERFACE_INCLUDE_DIRECTORIES)
+  foreach(directory ${DIRECTORIES})
+    file(MAKE_DIRECTORY ${directory})
+  endforeach()
 endif()
 
 find_package(dispatch CONFIG QUIET)
@@ -126,7 +107,7 @@ if(BUILD_TESTING)
   add_subdirectory(Tests)
 endif()
 
-if(BUILD_X10 AND NOT X10_FOUND AND NOT USE_BUNDLED_X10)
+if(NOT X10_FOUND AND NOT USE_BUNDLED_X10)
   get_swift_host_os(host_os)
   install(FILES ${SOURCE_DIR}/bazel-bin/tensorflow/compiler/tf2xla/xla_tensor/${CMAKE_SHARED_LIBRARY_PREFIX}x10${CMAKE_SHARED_LIBRARY_SUFFIX}
     DESTINATION lib/swift/${host_os})
diff --git a/Dockerfile b/Dockerfile
@@ -56,7 +56,6 @@ RUN cmake
       -D CMAKE_BUILD_TYPE=Release                                               \
       -D CMAKE_INSTALL_PREFIX=/swift-tensorflow-toolchain/usr                   \
       -D CMAKE_Swift_COMPILER=/swift-tensorflow-toolchain/usr/bin/swiftc        \
-      -D BUILD_X10=YES                                                          \
       -G Ninja                                                                  \
       -S /swift-apis
 RUN cmake --build /BinaryCache/tensorflow-swift-apis --verbose
diff --git a/README.md b/README.md
@@ -127,7 +127,7 @@ This will build X10 as part of the build.  Ensure that you do not have the
 x10 modules in the toolchain that you are using to develop here.
 
 ```shell
-cmake -B out -D BUILD_X10=YES -G Ninja -S swift-apis
+cmake -B out -G Ninja -S swift-apis
 cmake --build out
 ```
 
@@ -136,7 +136,7 @@ using the bundled X10 in the toolchain using
 `-D USE_BUNDLED_X10=YES -D USE_BUNDLED_CTENSORFLOW=YES`:
 
 ```shell
-cmake -B out -D BUILD_X10=YES -D USE_BUNDLED_CTENSORFLOW=YES -D USE_BUNDLED_X10=YES -G Ninja -S swift-apis
+cmake -B out -D USE_BUNDLED_CTENSORFLOW=YES -D USE_BUNDLED_X10=YES -G Ninja -S swift-apis
 cmake --build out
 ```
 
diff --git a/Sources/CMakeLists.txt b/Sources/CMakeLists.txt
@@ -2,7 +2,5 @@ add_subdirectory(CTensorFlow)
 add_subdirectory(third_party/Experimental)
 add_subdirectory(Tensor)
 add_subdirectory(TensorFlow)
-if(BUILD_X10)
-  add_subdirectory(CX10)
-  add_subdirectory(x10)
-endif()
+add_subdirectory(CX10)
+add_subdirectory(x10)
diff --git a/Sources/CTensorFlow/CMakeLists.txt b/Sources/CTensorFlow/CMakeLists.txt
@@ -3,21 +3,16 @@ if(NOT USE_BUNDLED_CTENSORFLOW)
   target_include_directories(CTensorFlow INTERFACE
     ${CMAKE_CURRENT_SOURCE_DIR})
 endif()
-if(BUILD_X10)
-  target_link_directories(CTensorFlow INTERFACE
-    $<TARGET_FILE_DIR:x10>)
-  target_link_libraries(CTensorFlow INTERFACE
-    x10)
-else()
-  target_link_directories(CTensorFlow INTERFACE
-    $<TARGET_FILE_DIR:tensorflow>)
-  target_link_libraries(CTensorFlow INTERFACE
-    tensorflow)
-endif()
+target_link_directories(CTensorFlow INTERFACE
+  $<TARGET_FILE_DIR:x10>)
+target_link_libraries(CTensorFlow INTERFACE
+  x10)
+
+list(GET X10_INCLUDE_DIRS 0 X10_SOURCE_DIR)
 
 install(FILES module.modulemap CTensorFlow.h
   DESTINATION lib/swift/tensorflow)
-install(DIRECTORY $<TARGET_PROPERTY:tensorflow,INTERFACE_INCLUDE_DIRECTORIES>/tensorflow/c
+install(DIRECTORY ${X10_SOURCE_DIR}/tensorflow/c
   DESTINATION lib/swift/tensorflow
   FILES_MATCHING
     PATTERN c_api.h
diff --git a/Sources/TensorFlow/CMakeLists.txt b/Sources/TensorFlow/CMakeLists.txt
@@ -70,25 +70,20 @@ add_library(TensorFlow SHARED
   Optimizers/MomentumBased.swift
   Optimizers/Optimizer.swift
   Optimizers/SGD.swift)
-if (BUILD_X10)
-  target_sources(TensorFlow PRIVATE
-    ../x10/swift_bindings/apis/CrossReplicaSum.swift
-    ../x10/swift_bindings/apis/DeviceScope.swift
-    ../x10/swift_bindings/apis/RawOpsManual.swift
+target_sources(TensorFlow PRIVATE
+  ../x10/swift_bindings/apis/CrossReplicaSum.swift
+  ../x10/swift_bindings/apis/DeviceScope.swift
+  ../x10/swift_bindings/apis/RawOpsManual.swift
 
-    ../x10/swift_bindings/Device.swift
-    ../x10/swift_bindings/XLAScalarType.swift
-    ../x10/swift_bindings/XLATensor.swift)
-  target_compile_definitions(TensorFlow PRIVATE
-    USING_X10_BACKEND
-    DEFAULT_BACKEND_EAGER)
-  target_link_libraries(TensorFlow PRIVATE
-    CX10
-    x10)
-else()
-  target_sources(TensorFlow PRIVATE
-    Core/Device.swift)
-endif()
+  ../x10/swift_bindings/Device.swift
+  ../x10/swift_bindings/XLAScalarType.swift
+  ../x10/swift_bindings/XLATensor.swift)
+target_compile_definitions(TensorFlow PRIVATE
+  USING_X10_BACKEND
+  DEFAULT_BACKEND_EAGER)
+target_link_libraries(TensorFlow PRIVATE
+  CX10
+  x10)
 set_target_properties(TensorFlow PROPERTIES
   INTERFACE_INCLUDE_DIRECTORIES ${CMAKE_Swift_MODULE_DIRECTORY}
   OUTPUT_NAME swiftTensorFlow)
diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt
@@ -23,6 +23,4 @@ add_test(NAME SwiftAPIsTests
 set_tests_properties(SwiftAPIsTests PROPERTIES
   ENVIRONMENT LD_LIBRARY_PATH=$<TARGET_FILE_DIR:Tensor>:$<TARGET_FILE_DIR:TensorFlow>)
 
-if(BUILD_X10)
-  add_subdirectory(x10)
-endif()
+add_subdirectory(x10)
diff --git a/cmake/modules/FindTensorFlow.cmake b/cmake/modules/FindTensorFlow.cmake
