Merge branch 'master' into coverity_scan

Author: msakai

.travis.yml

@@ -57,6 +57,17 @@ install:
     - python retrieve_data.py
     - python gen_test_data.py
 before_script:
+    - |
+      echo 'Checking tools and libraries version:'
+      if [ "$TRAVIS_OS_NAME" = "linux" ]; then
+        apt list --installed
+        ldconfig -p
+      fi
+      if [ "$TRAVIS_OS_NAME" = "osx" ]; then
+        brew list --versions
+      fi
+      cmake --version
+      make --version
     - ls -R $HOME/mkl-dnn
 script:
     #- if [ -f cov-int/build-log.txt ]; then cat cov-int/build-log.txt; fi

.travis/install_mkldnn.sh

@@ -1,7 +1,7 @@
 if [ ! -d "$HOME/mkl-dnn/lib" ]; then
     git clone https://github.com/intel/mkl-dnn.git
     cd mkl-dnn
-    git checkout v0.14
+    git checkout v0.15
     cd scripts && bash ./prepare_mkl.sh && cd ..
     sed -i 's/add_subdirectory(examples)//g' CMakeLists.txt
     sed -i 's/add_subdirectory(tests)//g' CMakeLists.txt

BUILDING.md

@@ -0,0 +1,152 @@
+# Building Menoh
+
+## Prerequisites
+To build Menoh, you require the following toolchains:
+
+Unix:
+- CMake 3.1 or later
+- GCC 4.9 or later
+
+macOS (OSX):
+- TODO
+
+Windows:
+- git
+- Visual Studio 2015
+
+You also need to install the dependent libraries on your system:
+
+- [Protocol Buffers](https://developers.google.com/protocol-buffers/) 2.6.1 or later
+    - Building instructions are [here](https://github.com/protocolbuffers/protobuf/blob/master/src/README.md)
+- [MKL-DNN](https://github.com/intel/mkl-dnn) 0.14 or later (for `mkldnn` backend)
+    - Building instructions are [here](https://github.com/intel/mkl-dnn/blob/master/README.md#installation)
+
+You can install `protobuf` through the package manager instead of building it yourself. `mkl-dnn` package, unfortunatelly, is not available in many environments at the moment (except for `brew` in macOS).
+
+### Debian/Ubuntu
+```
+apt-get install gcc g++ cmake-data cmake libopencv-dev libprotobuf-dev protobuf-compiler
+
+# See the MKL-DNN's instructions for details
+git clone https://github.com/intel/mkl-dnn.git
+cd mkl-dnn
+cd scripts && ./prepare_mkl.sh && cd ..
+mkdir -p build && cd build && cmake .. && make
+make install # as root
+```
+
+### macOS
+```
+brew update
+brew install protobuf mkl-dnn
+```
+
+### Windows
+
+None
+
+## Building
+
+### Unix
+Run the following command to build the source code:
+
+```
+git clone https://github.com/pfnet-research/menoh.git
+cd menoh
+mkdir -p build && cd build
+cmake ..
+make
+```
+
+To install Menoh into your system:
+
+```
+make install # as root
+```
+
+To run the example, you also need to download model data:
+
+```
+python retrieve_data.py
+```
+
+#### Static linking
+Menoh depends on several other libraries like `protobuf`. In Unix like systems, there is a chance to fail if you take the binary you built to other system because sometimes the dependent libraries installed on the system are not compatible with it.
+
+To improve the portability, you can statically link Menoh with its dependencies. There is the following options for `cmake` command:
+
+- `LINK_STATIC_LIBGCC` for `libgcc`
+- `LINK_STATIC_LIBSTDCXX` for `libstdc++`
+- `LINK_STATIC_LIBPROTOBUF` for `libprotobuf`
+- `LINK_STATIC_MKLDNN` for `libmkldnn` (NOT supported in this version)
+
+If you use `LINK_STATIC_LIBPROTOBUF` and `LINK_STATIC_MKLDNN`, you don't need to install the libraries on your system because they build static library from the source.
+
+All options are disabled by default, and you can turn them on as below:
+
+```
+cmake \
+  -DLINK_STATIC_LIBGCC=OFF \
+  -DLINK_STATIC_LIBSTDCXX=OFF \
+  -DLINK_STATIC_LIBPROTOBUF=ON \
+  ..
+```
+
+Note that static linking is great for binary portability, but it increases the binary size and potentially introduces further weird problems depending on the combination. We strongly recommend to avoid using `LINK_STATIC_*` options and consider building the binary against the specific system where you run your Menoh application.
+
+#### Old `libstdc++` ABI support
+Menoh has `USE_OLD_GLIBCXX_ABI` option to build its C++ source codes against the old `libstdc++` ABI to improve [backward compatibility](https://gcc.gnu.org/onlinedocs/libstdc++/manual/using_dual_abi.html) for the systems that depend on older GCC (`< 5.2`).
+
+Note that enabling this option may cause a problem if `protobuf` in your environment is compiled against the new ABI because its API includes `std::` data types. We recommend to use it along with `-DLINK_STATIC_LIBPROTOBUF=ON`:
+
+```
+cmake -DUSE_OLD_GLIBCXX_ABI=ON -DLINK_STATIC_LIBPROTOBUF=ON ..
+```
+
+### macOS (OS X)
+TODO
+
+### Windows
+
+Please specify your directory into (CMake_Install_Dir).
+
+#### mkl-dnn
+
+```
+git clone https://github.com/intel/mkl-dnn.git
+cd mkl-dnn/scripts
+.\prepare_mkl.bat
+cd ..
+mdir build
+cd build
+cmake .. -G "Visual Studio 14 Win64"  -DCMAKE_INSTALL_PREFIX=(CMake_Install_Dir)
+cmake --build . --config Debug --target install
+cmake --build . --config Release --target install
+cd ../..
+```
+
+#### protobuf
+
+Download and unzip https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protobuf-cpp-3.6.1.zip
+
+```
+cd protobuf-3.6.1/cmake
+mdir build
+cd build
+cmake .. -G "Visual Studio 14" -A x64 -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_INSTALL_PREFIX=(CMake_Install_Dir)
+cmake --build . --config Debug --target install
+cmake --build . --config Release --target install
+cd ../../..
+```
+
+#### menoh
+
+```
+git clone https://github.com/pfnet-research/menoh.git
+cd menoh
+mdir build
+cd build
+cmake .. -G "Visual Studio 14 Win64" -DCMAKE_PREFIX_PATH=CMake_Install_Dir) -DCMAKE_INSTALL_PREFIX=CMake_Install_Dir) -DENABLE_TEST=OFF -DENABLE_BENCHMARK=OFF -DENABLE_EXAMPLE=OFF -DENABLE_TOOL=OFF
+cmake --build . --config Debug --target install
+cmake --build . --config Release --target install
+```

CMakeLists.txt

@@ -4,14 +4,19 @@ set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
 
 set(MENOH_MAJOR_VERSION 1)
 set(MENOH_MINOR_VERSION 0)
-set(MENOH_PATCH_VERSION 2)
+set(MENOH_PATCH_VERSION 3)
 
 # Options
 option(BUILD_SHARED_LIBS "Build shared libs" ON)
+
+option(USE_OLD_GLIBCXX_ABI "Generate binaries for the old libstdc++ ABI" OFF)
+option(LINK_STATIC_LIBPROTOBUF "Link static libprotobuf to libmenoh" OFF)
+
 option(ENABLE_TEST "Build test" OFF)
 option(ENABLE_BENCHMARK "Build benchmark" ON)
 option(ENABLE_EXAMPLE "Build example" ON)
-option(ENABLE_TOOL "Build tool" ON)
+
+option(SHOW_ALL_VARIABLES "Debug: show all variables" OFF)
 
 # C++ setup
 set(CMAKE_CXX_STANDARD 14)
@@ -31,25 +36,36 @@ elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
     set(CMAKE_CXX_FLAGS_RELEASE "-O2 -s -DNDEBUG -march=native")
 endif()
 
+# Configure to use the new `libstdc++` ABI
+if(USE_OLD_GLIBCXX_ABI)
+    message(STATUS "Set _GLIBCXX_USE_CXX11_ABI macro to 0")
+    add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0)
+endif()
+
 include_directories("${PROJECT_SOURCE_DIR}")
 include_directories("${PROJECT_SOURCE_DIR}/include")
 
 set(EXTERNAL_DIR ${CMAKE_SOURCE_DIR}/external)
+set(DOWNLOAD_LOCATION ${CMAKE_CURRENT_BINARY_DIR}/downloads CACHE PATH "A directory to save the downloaded files.")
+mark_as_advanced(DOWNLOAD_LOCATION)
+
+# Enable ExternalProject_Add
+include(ExternalProject)
+
+# Setup protobuf
+include(SetupProtobuf)
 
-# ONNX setup
+# Generate source codes from ONNX protobuf schema
 set(ONNX_DIR ${EXTERNAL_DIR}/onnx)
-set(Protobuf_PROTOC_EXECUTABLE protoc CACHE STRING "protoc path")
-include(FindProtobuf)
-find_package(Protobuf 2.6.1 REQUIRED)
-include_directories(${PROTOBUF_INCLUDE_DIR})
 execute_process(COMMAND git submodule update --init -- ${ONNX_DIR} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
-execute_process(COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I=${ONNX_DIR} --cpp_out=${ONNX_DIR} ${ONNX_DIR}/onnx/onnx.proto WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+execute_process(COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} -I=${ONNX_DIR} --cpp_out=${ONNX_DIR} ${ONNX_DIR}/onnx/onnx.proto WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+
 set(ONNX_PROTO_HEADER ${ONNX_DIR}/onnx/onnx.pb.h)
 set(ONNX_PROTO_SRC ${ONNX_DIR}/onnx/onnx.pb.cc)
-include_directories(${EXTERNAL_DIR})
-include_directories(${ONNX_DIR})
 
-# MKLDNN setup
+include_directories(${ONNX_DIR}) # for ONNX_PROTO_HEADER
+
+# Setup MKLDNN
 find_package(MKLDNN "0.14")
 if (NOT MKLDNN_FOUND)
     message(FATAL_ERROR "MKLDNN is not found")
@@ -62,21 +78,32 @@ if(${ENABLE_TEST})
     # GTest setup
     set(GTEST_DIR test/lib/googletest)
     execute_process(COMMAND git submodule update --init -- ${GTEST_DIR} WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
+    message(STATUS "# add_subdirectory\(${GTEST_DIR}\)")
     add_subdirectory(${GTEST_DIR})
+    message(STATUS "# add_subdirectory\(test\)")
     add_subdirectory(test)
 endif()
 
 if(${ENABLE_BENCHMARK})
+    message(STATUS "# add_subdirectory\(benchmark\)")
     add_subdirectory(benchmark)
 endif()
 
 if(${ENABLE_EXAMPLE})
+    message(STATUS "# add_subdirectory\(example\)")
     add_subdirectory(example)
 endif()
 
-if(${ENABLE_TOOL})
-    add_subdirectory(tool)
-endif()
-
+message(STATUS "# add_subdirectory\(menoh\)")
 add_subdirectory(menoh)
+
+message(STATUS "# add_subdirectory\(include\)")
 add_subdirectory(include)
+
+if(SHOW_ALL_VARIABLES)
+    get_cmake_property(_variableNames VARIABLES)
+    list(SORT _variableNames)
+    foreach(_variableName ${_variableNames})
+        message(STATUS "${_variableName}=${${_variableName}}")
+    endforeach()
+endif()

README.md

@@ -27,6 +27,7 @@ This codebase contains C API and C++ API.
 - Go wrapper : [go-menoh](https://github.com/pfnet-research/go-menoh)
   - (unofficial wrapper [gomenoh](https://github.com/kou-m/gomenoh) by kou-m san has been merged)
 - Haskell wrapper : [menoh-haskell](https://github.com/pfnet-research/menoh-haskell)
+- Node.js wrapper : [node-menoh](https://github.com/pfnet-research/node-menoh)
 - Ruby wrapper : [menoh-ruby](https://github.com/pfnet-research/menoh-ruby)
 - Rust wrapper : [menoh-rs](https://github.com/pfnet-research/menoh-rs)
   - There is also [unofficial Rust wrapper by Y-Nak san](https://github.com/Y-Nak/menoh-rs)
@@ -47,7 +48,7 @@ This codebase contains C API and C++ API.
 
 ## Build
 
-Execute below commands in root directory.
+Execute following commands in root directory.
 
 ```
 python retrieve_data.py
@@ -56,33 +57,36 @@ cmake ..
 make
 ```
 
+See [BUILDING.md](BUILDING.md) for details.
+
 ## Installation
 
-Execute below command in build directory created at Build section.
+Execute following command in build directory created at Build section.
 
 ```
 make install
 ```
 
 # Run VGG16 example
 
-Execute below command in root directory.
+Execute following command in root directory.
 
 ```
 ./example/vgg16_example_in_cpp
 ```
 
-Result is below
+Result is here
 
 ```
 vgg16 example
--22.3708 -34.4082 -10.218 24.2962 -0.252342 -8.004 -27.0804 -23.0728 -7.05607 16.1343
+-23.149 -26.9167 -12.9911 10.3262 -0.845882 0.454192 -24.1435 -21.891 -3.98805 15.6863 
 top 5 categories are
-8 0.96132 n01514859 hen
-7 0.0369939 n01514668 cock
-86 0.00122795 n01807496 partridge
-82 0.000225824 n01797886 ruffed grouse, partridge, Bonasa umbellus
-97 3.83677e-05 n01847000 drake
+8 0.946072 n01514859 hen
+7 0.0485334 n01514668 cock
+86 0.00296125 n01807496 partridge
+82 0.000877083 n01797886 ruffed grouse, partridge, Bonasa umbellus
+97 0.000424042 n01847000 drake
+
 ```
 
 Please give `--help` option for details
@@ -96,7 +100,7 @@ Please give `--help` option for details
 
 Setup chainer
 
-Then, execute below commands in root directory.
+Then, execute following commands in root directory.
 
 ```
 python gen_test_data.py

appveyor.yml

@@ -24,12 +24,11 @@ install:
 
 - if [%TARGET%]==[mingw] (
     pacman -S --needed --noconfirm mingw-w64-x86_64-toolchain mingw-w64-x86_64-cmake mingw-w64-x86_64-opencv mingw-w64-x86_64-protobuf mingw-w64-x86_64-python3 mingw-w64-x86_64-python3-pip mingw-w64-x86_64-python3-numpy mingw-w64-x86_64-hdf5 mingw-w64-x86_64-ca-certificates &&
-    curl -omingw-w64-x86_64-mkl-dnn-0.14-3-x86_64.pkg.tar.xz -L https://preferredjp.box.com/shared/static/eb90vngoz8zngp6yy69x91q0swp11ha1.xz &&
-    pacman -U --noconfirm mingw-w64-x86_64-mkl-dnn-0.14-3-x86_64.pkg.tar.xz
+    curl -omingw-w64-x86_64-mkl-dnn-0.15-1-x86_64.pkg.tar.xz -L https://github.com/pfnet-research/menoh/releases/download/v1.0.3/mingw-w64-x86_64-mkl-dnn-0.15-1-x86_64.pkg.tar.xz &&
+    pacman -U --noconfirm mingw-w64-x86_64-mkl-dnn-0.15-1-x86_64.pkg.tar.xz
   ) else (
-    curl -omkl-dnn-0.14-win64.7z -L --insecure https://preferredjp.box.com/shared/static/njn6vrjt8zaslk82yjw7j1pczkcp3c0f.7z &&
-    mkdir mkl-dnn-0.14-win64 &&
-    7z x -omkl-dnn-0.14-win64 mkl-dnn-0.14-win64.7z &&
+    curl -omkl-dnn-0.15-win64.zip -L --insecure https://github.com/pfnet-research/menoh/releases/download/v1.0.3/mkl-dnn-0.15-win64.zip &&
+    7z x mkl-dnn-0.15-win64.zip &&
     call appveyor\install_protoc_msvc.bat
   )
 
@@ -45,10 +44,18 @@ install:
     python gen_test_data.py
   )
 
+before_build:
+  - if [%TARGET%]==[mingw] (
+      echo 'Checking tools and libraries version:' &&
+      pacman -Q &&
+      cmake --version &&
+      make --version
+    )
+
 build_script:
 - mkdir build
 - cd build
-- if [%TARGET%]==[msvc] set PATH=%PATH%;c:\protobuf-3.6.0-msvc\bin;c:\protobuf-3.6.0-msvc\include;c:\protobuf-3.6.0-msvc\lib;c:\projects\menoh\mkl-dnn-0.14-win64\bin;c:\projects\menoh\mkl-dnn-0.14-win64\include;c:\projects\menoh\mkl-dnn-0.14-win64\lib
+- if [%TARGET%]==[msvc] set PATH=%PATH%;c:\protobuf-3.6.0-msvc\bin;c:\protobuf-3.6.0-msvc\include;c:\protobuf-3.6.0-msvc\lib;c:\projects\menoh\mkl-dnn-0.15-win64\bin;c:\projects\menoh\mkl-dnn-0.15-win64\include;c:\projects\menoh\mkl-dnn-0.15-win64\lib
 - if [%TARGET%]==[mingw] (
     cmake -G "MSYS Makefiles" -DENABLE_TEST=ON -DCMAKE_INSTALL_PREFIX=/mingw64 .. &&
     make