diff --git a/.clang-format b/.clang-format index 1841c036f9..b8e9225290 100644 --- a/.clang-format +++ b/.clang-format @@ -23,3 +23,4 @@ AllowShortLoopsOnASingleLine: false ConstructorInitializerAllOnOneLineOrOnePerLine: false Cpp11BracedListStyle: false IndentCaseLabels: false +DerivePointerBinding: false diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 0000000000..357f046094 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,19 @@ +--- +Checks: ' + ,readability-avoid-const-params-in-decls, + ,readability-inconsistent-declaration-parameter-name, + ,readability-non-const-parameter, + ,readability-redundant-string-cstr, + ,readability-redundant-string-init, + ,readability-simplify-boolean-expr, + ,cppcoreguidelines-pro-type-cstyle-cast, +' +WarningsAsErrors: ' + ,readability-avoid-const-params-in-decls, + ,readability-inconsistent-declaration-parameter-name, + ,readability-non-const-parameter, + ,readability-redundant-string-cstr, + ,readability-redundant-string-init, + ,readability-simplify-boolean-expr, + ,cppcoreguidelines-pro-type-cstyle-cast, +' diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..0cc68d65ac --- /dev/null +++ b/.editorconfig @@ -0,0 +1,11 @@ +root = true + +[*] +charset = utf-8 +indent_style = space +indent_size = 2 +insert_final_newline = true +end_of_line = lf + +[CMakeLists.txt] +indent_style = tab diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..6fddca0d6e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/linux-musl.yml b/.github/workflows/linux-musl.yml new file mode 100644 index 0000000000..5f049b12e7 --- /dev/null +++ b/.github/workflows/linux-musl.yml @@ -0,0 +1,68 @@ +name: ci-linux-musl + +on: + workflow_dispatch: + pull_request: + push: + release: + types: [published] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: {} + +jobs: + build: + runs-on: ubuntu-24.04 + container: alpine:edge + permissions: + contents: read + strategy: + fail-fast: false + matrix: + build_method: ["python", "cmake"] + + steps: + - name: Host - checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Install ninja build optional dependencies + run: apk update && apk add -u --no-cache python3 build-base cmake re2c + + - name: Configure ninja build + if: matrix.build_method == 'cmake' + run: cmake -B build -D CMAKE_BUILD_TYPE="Release" + + - name: Cmake Build ninja + if: matrix.build_method == 'cmake' + run: cmake --build build --parallel --config Release + + - name: Cmake test ninja + if: matrix.build_method == 'cmake' + run: build/ninja_test --gtest_color=yes + + - name: Python Build ninja + if: matrix.build_method == 'python' + run: python3 configure.py --bootstrap --verbose + + - name: Python test ninja + if: matrix.build_method == 'python' + run: | + ./ninja all + python3 misc/ninja_syntax_test.py + # python3 misc/output_test.py + + - name: Move ninja binary + if: matrix.build_method == 'cmake' + run: mv -f build/ninja ninja + + - name: ninja-ninja --version + run: ./ninja --version >> $GITHUB_STEP_SUMMARY + + - name: binary info via file + run: file ./ninja >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml new file mode 100644 index 0000000000..3b16146317 --- /dev/null +++ b/.github/workflows/linux.yml @@ -0,0 +1,215 @@ +name: Linux + +on: + pull_request: + push: + release: + types: published + +jobs: + fedora: + runs-on: [ubuntu-latest] + container: + image: fedora:40 + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: dnf install -y ninja-build cmake gtest-devel re2c clang util-linux clang-tools-extra + - name: Linting + run: misc/ci.py + - name: Configure with CMake + run: cmake -Bbuild -G"Ninja Multi-Config" -DNINJA_CLANG_TIDY=1 + - name: Build debug ninja + run: CLICOLOR_FORCE=1 ninja + working-directory: build + - name: Test debug ninja + working-directory: build/Debug + run: | + ./ninja_test --gtest_color=yes + ../../misc/output_test.py + - name: Build release ninja + run: CLICOLOR_FORCE=1 ninja -f build-Release.ninja + working-directory: build + - name: Test release ninja + working-directory: build/Release + run: | + ./ninja_test --gtest_color=yes + ../../misc/output_test.py + + build: + runs-on: [ubuntu-latest] + container: + image: rockylinux:8 + steps: + - uses: actions/checkout@v4 + - uses: codespell-project/actions-codespell@master + with: + ignore_words_list: fo,wee,addin,notin + - name: Install dependencies + run: | + dnf install -y make gcc-c++ libasan clang-analyzer cmake dnf-plugins-core epel-release + dnf config-manager --set-enabled powertools + dnf install -y gtest-devel p7zip p7zip-plugins ninja-build + + - name: Build debug ninja + shell: bash + env: + CFLAGS: -fstack-protector-all -fsanitize=address + CXXFLAGS: -fstack-protector-all -fsanitize=address + run: | + scan-build -o scanlogs cmake -GNinja -DCMAKE_BUILD_TYPE=Debug -B debug-build + scan-build -o scanlogs cmake --build debug-build --parallel --config Debug + + - name: Test debug ninja + run: ASAN_OPTIONS=detect_leaks=0 ./ninja_test + working-directory: debug-build + + - name: Build release ninja + shell: bash + run: | + cmake -GNinja -DCMAKE_BUILD_TYPE=Release -B release-build + cmake --build release-build --parallel --config Release + strip release-build/ninja + + - name: Test release ninja + run: ./ninja_test + working-directory: release-build + + - name: Create ninja archive + run: | + mkdir artifact + 7z a artifact/ninja-linux.zip ./release-build/ninja + + # Upload ninja binary archive as an artifact + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ninja-binary-archives + path: artifact + + - name: Upload release asset + if: github.event.action == 'published' + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./artifact/ninja-linux.zip + asset_name: ninja-linux.zip + asset_content_type: application/zip + + test: + runs-on: [ubuntu-latest] + container: + image: ubuntu:20.04 + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + apt update + apt install -y python3-pytest ninja-build python3-pip clang libgtest-dev + pip3 install cmake==3.17.* + - name: Configure (GCC) + run: cmake -Bbuild-gcc -DCMAKE_BUILD_TYPE=Debug -G'Ninja Multi-Config' + + - name: Build (GCC, Debug) + run: cmake --build build-gcc --config Debug + - name: Unit tests (GCC, Debug) + run: ./build-gcc/Debug/ninja_test + - name: Python tests (GCC, Debug) + run: pytest-3 --color=yes ../.. + working-directory: build-gcc/Debug + + - name: Build (GCC, Release) + run: cmake --build build-gcc --config Release + - name: Unit tests (GCC, Release) + run: ./build-gcc/Release/ninja_test + - name: Python tests (GCC, Release) + run: pytest-3 --color=yes ../.. + working-directory: build-gcc/Release + + - name: Configure (Clang) + run: CC=clang CXX=clang++ cmake -Bbuild-clang -DCMAKE_BUILD_TYPE=Debug -G'Ninja Multi-Config' + + - name: Build (Clang, Debug) + run: cmake --build build-clang --config Debug + - name: Unit tests (Clang, Debug) + run: ./build-clang/Debug/ninja_test + - name: Python tests (Clang, Debug) + run: pytest-3 --color=yes ../.. + working-directory: build-clang/Debug + + - name: Build (Clang, Release) + run: cmake --build build-clang --config Release + - name: Unit tests (Clang, Release) + run: ./build-clang/Release/ninja_test + - name: Python tests (Clang, Release) + run: pytest-3 --color=yes ../.. + working-directory: build-clang/Release + + build-with-python: + runs-on: [ubuntu-latest] + container: + image: ${{ matrix.image }} + strategy: + matrix: + image: ["ubuntu:20.04", "ubuntu:22.04", "ubuntu:24.04"] + steps: + - uses: actions/checkout@v4 + - name: Install dependencies + run: | + apt update + apt install -y g++ python3 + - name: ${{ matrix.image }} + run: | + python3 configure.py --bootstrap + ./ninja all + python3 misc/ninja_syntax_test.py + ./misc/output_test.py + + build-aarch64: + name: Build Linux ARM64 + runs-on: [ubuntu-24.04-arm] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - run: | + sudo apt-get update -q -y + sudo apt-get install -q -y make gcc g++ libasan5 clang-tools curl p7zip-full file cmake re2c + + - run: | + # BUILD + cmake -DCMAKE_BUILD_TYPE=Release -B release-build + cmake --build release-build --parallel --config Release + strip release-build/ninja + file release-build/ninja + + # TEST + pushd release-build + ./ninja_test + popd + + # CREATE ARCHIVE + mkdir artifact + 7z a artifact/ninja-linux-aarch64.zip ./release-build/ninja + + # Upload ninja binary archive as an artifact + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ninja-aarch64-binary-archives + path: artifact + + - name: Upload release asset + if: github.event.action == 'published' + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./artifact/ninja-linux-aarch64.zip + asset_name: ninja-linux-aarch64.zip + asset_content_type: application/zip diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml new file mode 100644 index 0000000000..33acd892eb --- /dev/null +++ b/.github/workflows/macos.yml @@ -0,0 +1,53 @@ +name: macOS + +on: + pull_request: + push: + release: + types: published + +jobs: + build: + runs-on: macos-13 + + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: brew install re2c p7zip cmake + + - name: Build ninja + shell: bash + env: + MACOSX_DEPLOYMENT_TARGET: 10.15 + run: | + cmake -Bbuild -GXcode '-DCMAKE_OSX_ARCHITECTURES=arm64;x86_64' + cmake --build build --config Release + + - name: Test ninja (Release) + run: ./ninja_test + working-directory: build/Release + + - name: Create ninja archive + shell: bash + run: | + mkdir artifact + 7z a artifact/ninja-mac.zip ./build/Release/ninja + + # Upload ninja binary archive as an artifact + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ninja-binary-archives + path: artifact + + - name: Upload release asset + if: github.event.action == 'published' + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./artifact/ninja-mac.zip + asset_name: ninja-mac.zip + asset_content_type: application/zip diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml new file mode 100644 index 0000000000..c20889d40d --- /dev/null +++ b/.github/workflows/windows.yml @@ -0,0 +1,67 @@ +name: Windows + +on: + pull_request: + push: + release: + types: published + +jobs: + build: + runs-on: windows-latest + + strategy: + fail-fast: false + matrix: + include: + - arch: 'x64' + suffix: '' + - arch: 'arm64' + suffix: 'arm64' + + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: choco install re2c + + - name: Build ninja + shell: bash + run: | + cmake -Bbuild -A ${{ matrix.arch }} + cmake --build build --parallel --config Debug + cmake --build build --parallel --config Release + + - name: Test ninja (Debug) + if: matrix.arch != 'arm64' + run: .\ninja_test.exe + working-directory: build/Debug + + - name: Test ninja (Release) + if: matrix.arch != 'arm64' + run: .\ninja_test.exe + working-directory: build/Release + + - name: Create ninja archive + shell: bash + run: | + mkdir artifact + 7z a artifact/ninja-win${{ matrix.suffix }}.zip ./build/Release/ninja.exe + + # Upload ninja binary archive as an artifact + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ninja-binary-archives${{ matrix.suffix }} + path: artifact + + - name: Upload release asset + if: github.event.action == 'published' + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ github.event.release.upload_url }} + asset_path: ./artifact/ninja-win${{ matrix.suffix }}.zip + asset_name: ninja-win${{ matrix.suffix }}.zip + asset_content_type: application/zip diff --git a/.gitignore b/.gitignore index 46736a6915..bb13dcd3f4 100644 --- a/.gitignore +++ b/.gitignore @@ -3,14 +3,14 @@ *.exe *.pdb *.ilk -TAGS -/build +/build*/ /build.ninja /ninja /ninja.bootstrap /build_log_perftest /canon_perftest /clparser_perftest +/elide_middle_perftest /depfile_parser_perftest /hash_collision_bench /ninja_test @@ -18,8 +18,8 @@ TAGS /graph.png /doc/manual.html /doc/doxygen -/gtest-1.6.0 *.patch +.DS_Store # Eclipse project files .project @@ -36,3 +36,15 @@ TAGS # Visual Studio Code project files /.vscode/ /.ccls-cache/ + +# Qt Creator project files +/CMakeLists.txt.user + +# clangd +/.clangd/ +/compile_commands.json +/.cache/ + +# Visual Studio files +/.vs/ +/out/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f76b9829c5..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,16 +0,0 @@ -matrix: - include: - - os: linux - compiler: gcc - - os: linux - compiler: clang - - os: osx -sudo: false -language: cpp -script: - - ./misc/ci.py - - ./configure.py --bootstrap - - ./ninja all - - ./ninja_test --gtest_filter=-SubprocessTest.SetWithLots - - ./misc/ninja_syntax_test.py - - ./misc/output_test.py diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000000..881dd54ff9 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,327 @@ +cmake_minimum_required(VERSION 3.15) + +include(CheckSymbolExists) +include(CheckIPOSupported) + +option(NINJA_BUILD_BINARY "Build ninja binary" ON) +option(NINJA_FORCE_PSELECT "Use pselect() even on platforms that provide ppoll()" OFF) + +project(ninja CXX) + +# --- optional link-time optimization +check_ipo_supported(RESULT lto_supported OUTPUT error) + +if(lto_supported) + message(STATUS "IPO / LTO enabled") + set(CMAKE_POLICY_DEFAULT_CMP0069 NEW) + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE TRUE) +else() + message(STATUS "IPO / LTO not supported: <${error}>") +endif() + +# --- compiler flags +if(MSVC) + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") + string(REPLACE "/GR" "" CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) + # Note that these settings are separately specified in configure.py, and + # these lists should be kept in sync. + add_compile_options(/W4 /wd4100 /wd4267 /wd4706 /wd4702 /wd4244 /GR- /Zc:__cplusplus) + add_compile_definitions(_CRT_SECURE_NO_WARNINGS) +else() + include(CheckCXXCompilerFlag) + check_cxx_compiler_flag(-Wno-deprecated flag_no_deprecated) + if(flag_no_deprecated) + add_compile_options(-Wno-deprecated) + endif() + if(CMAKE_VERSION VERSION_LESS 3.24) + check_cxx_compiler_flag(-fdiagnostics-color flag_color_diag) + if(flag_color_diag) + add_compile_options(-fdiagnostics-color) + endif() + elseif(NOT DEFINED ENV{CMAKE_COLOR_DIAGNOSTICS}) + set(CMAKE_COLOR_DIAGNOSTICS ON) + endif() + + if(NOT NINJA_FORCE_PSELECT) + # Check whether ppoll() is usable on the target platform. + # Set -DUSE_PPOLL=1 if this is the case. + # + # NOTE: Use check_cxx_symbol_exists() instead of check_symbol_exists() + # because on Linux, only exposes the symbol when _GNU_SOURCE + # is defined. + # + # Both g++ and clang++ define the symbol by default, because the C++ + # standard library headers require it, but *not* gcc and clang, which + # are used by check_symbol_exists(). + include(CheckCXXSymbolExists) + check_cxx_symbol_exists(ppoll poll.h HAVE_PPOLL) + if(HAVE_PPOLL) + add_compile_definitions(USE_PPOLL=1) + endif() + endif() +endif() + +# --- optional re2c +set(RE2C_MAJOR_VERSION 0) +find_program(RE2C re2c) +if(RE2C) + execute_process(COMMAND "${RE2C}" --vernum OUTPUT_VARIABLE RE2C_RAW_VERSION) + math(EXPR RE2C_MAJOR_VERSION "${RE2C_RAW_VERSION} / 10000") +endif() +if(${RE2C_MAJOR_VERSION} GREATER 1) + # the depfile parser and ninja lexers are generated using re2c. + function(re2c IN OUT) + add_custom_command(DEPENDS ${IN} OUTPUT ${OUT} + COMMAND ${RE2C} -b -i --no-generation-date --no-version -o ${OUT} ${IN} + ) + endfunction() + re2c(${PROJECT_SOURCE_DIR}/src/depfile_parser.in.cc ${PROJECT_BINARY_DIR}/depfile_parser.cc) + re2c(${PROJECT_SOURCE_DIR}/src/lexer.in.cc ${PROJECT_BINARY_DIR}/lexer.cc) + add_library(libninja-re2c OBJECT ${PROJECT_BINARY_DIR}/depfile_parser.cc ${PROJECT_BINARY_DIR}/lexer.cc) +else() + message(WARNING "re2c 2 or later was not found; changes to src/*.in.cc will not affect your build.") + add_library(libninja-re2c OBJECT src/depfile_parser.cc src/lexer.cc) +endif() +target_include_directories(libninja-re2c PRIVATE src) + +# --- Check for 'browse' mode support +function(check_platform_supports_browse_mode RESULT) + # Make sure the inline.sh script works on this platform. + # It uses the shell commands such as 'od', which may not be available. + + execute_process( + COMMAND sh -c "echo 'TEST' | src/inline.sh var" + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + RESULT_VARIABLE inline_result + OUTPUT_QUIET + ERROR_QUIET + ) + if(NOT inline_result EQUAL "0") + # The inline script failed, so browse mode is not supported. + set(${RESULT} "0" PARENT_SCOPE) + if(NOT WIN32) + message(WARNING "browse feature omitted due to inline script failure") + endif() + return() + endif() + + # Now check availability of the unistd header + check_symbol_exists(fork "unistd.h" HAVE_FORK) + check_symbol_exists(pipe "unistd.h" HAVE_PIPE) + set(browse_supported 0) + if (HAVE_FORK AND HAVE_PIPE) + set(browse_supported 1) + endif () + set(${RESULT} "${browse_supported}" PARENT_SCOPE) + if(NOT browse_supported) + message(WARNING "browse feature omitted due to missing `fork` and `pipe` functions") + endif() + +endfunction() + +set(NINJA_PYTHON "python" CACHE STRING "Python interpreter to use for the browse tool") + +check_platform_supports_browse_mode(platform_supports_ninja_browse) + +# Core source files all build into ninja library. +add_library(libninja OBJECT + src/build_log.cc + src/build.cc + src/clean.cc + src/clparser.cc + src/dyndep.cc + src/dyndep_parser.cc + src/debug_flags.cc + src/deps_log.cc + src/disk_interface.cc + src/edit_distance.cc + src/elide_middle.cc + src/eval_env.cc + src/graph.cc + src/graphviz.cc + src/json.cc + src/line_printer.cc + src/manifest_parser.cc + src/metrics.cc + src/missing_deps.cc + src/parser.cc + src/real_command_runner.cc + src/state.cc + src/status_printer.cc + src/string_piece_util.cc + src/util.cc + src/version.cc +) +if(WIN32) + target_sources(libninja PRIVATE + src/subprocess-win32.cc + src/includes_normalize-win32.cc + src/msvc_helper-win32.cc + src/msvc_helper_main-win32.cc + src/getopt.c + src/minidump-win32.cc + ) + # Build getopt.c, which can be compiled as either C or C++, as C++ + # so that build environments which lack a C compiler, but have a C++ + # compiler may build ninja. + set_source_files_properties(src/getopt.c PROPERTIES LANGUAGE CXX) + + # windows.h defines min() and max() which conflict with std::min() + # and std::max(), which both might be used in sources. Avoid compile + # errors by telling windows.h to not define those two. + add_compile_definitions(NOMINMAX) +else() + target_sources(libninja PRIVATE src/subprocess-posix.cc) + if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX") + target_sources(libninja PRIVATE src/getopt.c) + # Build getopt.c, which can be compiled as either C or C++, as C++ + # so that build environments which lack a C compiler, but have a C++ + # compiler may build ninja. + set_source_files_properties(src/getopt.c PROPERTIES LANGUAGE CXX) + endif() + + # Needed for perfstat_cpu_total + if(CMAKE_SYSTEM_NAME STREQUAL "AIX") + target_link_libraries(libninja PUBLIC "-lperfstat") + endif() +endif() + +target_compile_features(libninja PUBLIC cxx_std_11) +target_compile_features(libninja-re2c PUBLIC cxx_std_11) + +#Fixes GetActiveProcessorCount on MinGW +if(MINGW) +target_compile_definitions(libninja PRIVATE _WIN32_WINNT=0x0601 __USE_MINGW_ANSI_STDIO=1) +endif() + +# On IBM i (identified as "OS400" for compatibility reasons) and AIX, this fixes missing +# PRId64 (and others) at compile time in C++ sources +if(CMAKE_SYSTEM_NAME STREQUAL "OS400" OR CMAKE_SYSTEM_NAME STREQUAL "AIX") + add_compile_definitions(__STDC_FORMAT_MACROS) +endif() + +# Main executable is library plus main() function. +if(NINJA_BUILD_BINARY) + add_executable(ninja src/ninja.cc) + target_link_libraries(ninja PRIVATE libninja libninja-re2c) + + if(WIN32) + target_sources(ninja PRIVATE windows/ninja.manifest) + endif() + + option(NINJA_CLANG_TIDY "Run clang-tidy on source files" OFF) + if(NINJA_CLANG_TIDY) + set_target_properties(libninja PROPERTIES CXX_CLANG_TIDY "clang-tidy;--use-color") + set_target_properties(ninja PROPERTIES CXX_CLANG_TIDY "clang-tidy;--use-color") + endif() +endif() + +# Adds browse mode into the ninja binary if it's supported by the host platform. +if(platform_supports_ninja_browse) + # Inlines src/browse.py into the browse_py.h header, so that it can be included + # by src/browse.cc + add_custom_command( + OUTPUT build/browse_py.h + MAIN_DEPENDENCY src/browse.py + DEPENDS src/inline.sh + COMMAND ${CMAKE_COMMAND} -E make_directory ${PROJECT_BINARY_DIR}/build + COMMAND src/inline.sh kBrowsePy + < src/browse.py + > ${PROJECT_BINARY_DIR}/build/browse_py.h + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + VERBATIM + ) + + if(NINJA_BUILD_BINARY) + target_compile_definitions(ninja PRIVATE NINJA_HAVE_BROWSE) + target_sources(ninja PRIVATE src/browse.cc) + endif() + set_source_files_properties(src/browse.cc + PROPERTIES + OBJECT_DEPENDS "${PROJECT_BINARY_DIR}/build/browse_py.h" + INCLUDE_DIRECTORIES "${PROJECT_BINARY_DIR}" + COMPILE_DEFINITIONS NINJA_PYTHON="${NINJA_PYTHON}" + ) +endif() + +include(CTest) +if(BUILD_TESTING) + + # Can be removed if cmake min version is >=3.24 + if (POLICY CMP0135) + cmake_policy(SET CMP0135 NEW) + endif() + + find_package(GTest) + if(NOT GTest_FOUND) + include(FetchContent) + FetchContent_Declare( + googletest + URL https://github.com/google/googletest/archive/refs/tags/release-1.12.1.tar.gz + URL_HASH SHA256=81964fe578e9bd7c94dfdb09c8e4d6e6759e19967e397dbea48d1c10e45d0df2 + ) + FetchContent_MakeAvailable(googletest) + endif() + + # Tests all build into ninja_test executable. + add_executable(ninja_test + src/build_log_test.cc + src/build_test.cc + src/clean_test.cc + src/clparser_test.cc + src/depfile_parser_test.cc + src/deps_log_test.cc + src/disk_interface_test.cc + src/dyndep_parser_test.cc + src/edit_distance_test.cc + src/elide_middle_test.cc + src/explanations_test.cc + src/graph_test.cc + src/json_test.cc + src/lexer_test.cc + src/manifest_parser_test.cc + src/missing_deps_test.cc + src/ninja_test.cc + src/state_test.cc + src/string_piece_util_test.cc + src/subprocess_test.cc + src/test.cc + src/util_test.cc + ) + if(WIN32) + target_sources(ninja_test PRIVATE src/includes_normalize_test.cc src/msvc_helper_test.cc + windows/ninja.manifest) + + if(MSVC) + # Silence warnings about using unlink rather than _unlink + target_compile_definitions(ninja_test PRIVATE _CRT_NONSTDC_NO_DEPRECATE) + endif() + endif() + find_package(Threads REQUIRED) + target_link_libraries(ninja_test PRIVATE libninja libninja-re2c GTest::gtest Threads::Threads) + + foreach(perftest + build_log_perftest + canon_perftest + clparser_perftest + depfile_parser_perftest + elide_middle_perftest + hash_collision_bench + manifest_parser_perftest + ) + add_executable(${perftest} src/${perftest}.cc) + target_link_libraries(${perftest} PRIVATE libninja libninja-re2c) + endforeach() + + if(CMAKE_SYSTEM_NAME STREQUAL "AIX" AND CMAKE_SIZEOF_VOID_P EQUAL 4) + # These tests require more memory than will fit in the standard AIX shared stack/heap (256M) + target_link_options(hash_collision_bench PRIVATE "-Wl,-bmaxdata:0x80000000") + target_link_options(manifest_parser_perftest PRIVATE "-Wl,-bmaxdata:0x80000000") + endif() + + add_test(NAME NinjaTest COMMAND ninja_test) +endif() + +if(NINJA_BUILD_BINARY) + install(TARGETS ninja) +endif() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..37f6ebc3fd --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,30 @@ +# How to successfully make changes to Ninja + +We're very wary of changes that increase the complexity of Ninja (in particular, +new build file syntax or command-line flags) or increase the maintenance burden +of Ninja. Ninja is already successfully used by hundreds of developers for large +projects and it already achieves (most of) the goals we set out for it to do. +It's probably best to discuss new feature ideas on the +[mailing list](https://groups.google.com/forum/#!forum/ninja-build) or in an +issue before creating a PR. + +## Coding guidelines + +Generally it's the +[Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html) with +a few additions: + +* We have used `using namespace std;` a lot in the past. For new contributions, + please try to avoid relying on it and instead whenever possible use `std::`. + However, please do not change existing code simply to add `std::` unless your + contribution already needs to change that line of code anyway. +* Use `///` for [Doxygen](http://www.doxygen.nl/) (use `\a` to refer to + arguments). +* It's not necessary to document each argument, especially when they're + relatively self-evident (e.g. in + `CanonicalizePath(string* path, string* err)`, the arguments are hopefully + obvious). + +If you're unsure about code formatting, please use +[clang-format](https://clang.llvm.org/docs/ClangFormat.html). However, please do +not format code that is not otherwise part of your contribution. diff --git a/HACKING.md b/HACKING.md deleted file mode 100644 index bd6fec7d18..0000000000 --- a/HACKING.md +++ /dev/null @@ -1,252 +0,0 @@ -## Basic overview - -`./configure.py` generates the `build.ninja` files used to build -ninja. It accepts various flags to adjust build parameters. -Run './configure.py --help' for more configuration options. - -The primary build target of interest is `ninja`, but when hacking on -Ninja your changes should be testable so it's more useful to build and -run `ninja_test` when developing. - -### Bootstrapping - -Ninja is built using itself. To bootstrap the first binary, run the -configure script as `./configure.py --bootstrap`. This first compiles -all non-test source files together, then re-builds Ninja using itself. -You should end up with a `ninja` binary (or `ninja.exe`) in the project root. - -#### Windows - -On Windows, you'll need to install Python to run `configure.py`, and -run everything under a Visual Studio Tools Command Prompt (or after -running `vcvarsall` in a normal command prompt). - -For other combinations such as gcc/clang you will need the compiler -(gcc/cl) in your PATH and you will have to set the appropriate -platform configuration script. - -See below if you want to use mingw or some other compiler instead of -Visual Studio. - -##### Using Visual Studio -Assuming that you now have Python installed, then the steps for building under -Windows using Visual Studio are: - -Clone and checkout the latest release (or whatever branch you want). You -can do this in either a command prompt or by opening a git bash prompt: - -``` - $ git clone git://github.com/ninja-build/ninja.git && cd ninja - $ git checkout release -``` - -Then: - -1. Open a Windows command prompt in the folder where you checked out ninja. -2. Select the Microsoft build environment by running -`vcvarsall.bat` with the appropriate environment. -3. Build ninja and test it. - -The steps for a Visual Studio 2015 64-bit build are outlined here: - -``` - > "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64 - > python configure.py --bootstrap - > ninja --help -``` -Copy the ninja executable to another location, if desired, e.g. C:\local\Ninja. - -Finally add the path where ninja.exe is to the PATH variable. - -### Adjusting build flags - -Build in "debug" mode while developing (disables optimizations and builds -way faster on Windows): - - ./configure.py --debug - -To use clang, set `CXX`: - - CXX=clang++ ./configure.py - -## How to successfully make changes to Ninja - -Github pull requests are convenient for me to merge (I can just click -a button and it's all handled server-side), but I'm also comfortable -accepting pre-github git patches (via `send-email` etc.). - -Good pull requests have all of these attributes: - -* Are scoped to one specific issue -* Include a test to demonstrate their correctness -* Update the docs where relevant -* Match the Ninja coding style (see below) -* Don't include a mess of "oops, fix typo" commits - -These are typically merged without hesitation. If a change is lacking -any of the above I usually will ask you to fix it, though there are -obvious exceptions (fixing typos in comments don't need tests). - -I am very wary of changes that increase the complexity of Ninja (in -particular, new build file syntax or command-line flags) or increase -the maintenance burden of Ninja. Ninja is already successfully used -by hundreds of developers for large projects and it already achieves -(most of) the goals I set out for it to do. It's probably best to -discuss new feature ideas on the [mailing list](https://groups.google.com/forum/#!forum/ninja-build) -before I shoot down your patch. - -## Testing - -### Test-driven development - -Set your build command to - - ./ninja ninja_test && ./ninja_test --gtest_filter=MyTest.Name - -now you can repeatedly run that while developing until the tests pass -(I frequently set it as my compilation command in Emacs). Remember to -build "all" before committing to verify the other source still works! - -## Testing performance impact of changes - -If you have a Chrome build handy, it's a good test case. There's a -script at `misc/measure.py` that repeatedly runs a command (to address -variance) and summarizes its runtime. E.g. - - path/to/misc/measure.py path/to/my/ninja chrome - -For changing the depfile parser, you can also build `parser_perftest` -and run that directly on some representative input files. - -## Coding guidelines - -Generally it's the [Google C++ coding style][], but in brief: - -* Function name are camelcase. -* Member methods are camelcase, except for trivial getters which are - underscore separated. -* Local variables are underscore separated. -* Member variables are underscore separated and suffixed by an extra - underscore. -* Two spaces indentation. -* Opening braces is at the end of line. -* Lines are 80 columns maximum. -* All source files should have the Google Inc. license header. - -[Google C++ coding style]: https://google.github.io/styleguide/cppguide.html - -## Documentation - -### Style guidelines - -* Use `///` for doxygen. -* Use `\a` to refer to arguments. -* It's not necessary to document each argument, especially when they're - relatively self-evident (e.g. in `CanonicalizePath(string* path, string* err)`, - the arguments are hopefully obvious) - -### Building the manual - - sudo apt-get install asciidoc --no-install-recommends - ./ninja manual - -### Building the code documentation - - sudo apt-get install doxygen - ./ninja doxygen - -## Building for Windows - -While developing, it's helpful to copy `ninja.exe` to another name like -`n.exe`; otherwise, rebuilds will be unable to write `ninja.exe` because -it's locked while in use. - -### Via Visual Studio - -* Install Visual Studio (Express is fine), [Python for Windows][], - and (if making changes) googletest (see above instructions) -* In a Visual Studio command prompt: `python configure.py --bootstrap` - -[Python for Windows]: http://www.python.org/getit/windows/ - -### Via mingw on Windows (not well supported) - -* Install mingw, msys, and python -* In the mingw shell, put Python in your path, and - `python configure.py --bootstrap` -* To reconfigure, run `python configure.py` -* Remember to strip the resulting executable if size matters to you - -### Via mingw on Linux (not well supported) - -Setup on Ubuntu Lucid: -* `sudo apt-get install gcc-mingw32 wine` -* `export CC=i586-mingw32msvc-cc CXX=i586-mingw32msvc-c++ AR=i586-mingw32msvc-ar` - -Setup on Ubuntu Precise: -* `sudo apt-get install gcc-mingw-w64-i686 g++-mingw-w64-i686 wine` -* `export CC=i686-w64-mingw32-gcc CXX=i686-w64-mingw32-g++ AR=i686-w64-mingw32-ar` - -Setup on Arch: -* Uncomment the `[multilib]` section of `/etc/pacman.conf` and `sudo pacman -Sy`. -* `sudo pacman -S mingw-w64-gcc wine` -* `export CC=x86_64-w64-mingw32-cc CXX=x86_64-w64-mingw32-c++ AR=x86_64-w64-mingw32-ar` -* `export CFLAGS=-I/usr/x86_64-w64-mingw32/include` - -Then run: -* `./configure.py --platform=mingw --host=linux` -* Build `ninja.exe` using a Linux ninja binary: `/path/to/linux/ninja` -* Run: `./ninja.exe` (implicitly runs through wine(!)) - -### Using Microsoft compilers on Linux (extremely flaky) - -The trick is to install just the compilers, and not all of Visual Studio, -by following [these instructions][win7sdk]. - -[win7sdk]: http://www.kegel.com/wine/cl-howto-win7sdk.html - -### Using gcov - -Do a clean debug build with the right flags: - - CFLAGS=-coverage LDFLAGS=-coverage ./configure.py --debug - ninja -t clean ninja_test && ninja ninja_test - -Run the test binary to generate `.gcda` and `.gcno` files in the build -directory, then run gcov on the .o files to generate `.gcov` files in the -root directory: - - ./ninja_test - gcov build/*.o - -Look at the generated `.gcov` files directly, or use your favorite gcov viewer. - -### Using afl-fuzz - -Build with afl-clang++: - - CXX=path/to/afl-1.20b/afl-clang++ ./configure.py - ninja - -Then run afl-fuzz like so: - - afl-fuzz -i misc/afl-fuzz -o /tmp/afl-fuzz-out ./ninja -n -f @@ - -You can pass `-x misc/afl-fuzz-tokens` to use the token dictionary. In my -testing, that did not seem more effective though. - -#### Using afl-fuzz with asan - -If you want to use asan (the `isysroot` bit is only needed on OS X; if clang -can't find C++ standard headers make sure your LLVM checkout includes a libc++ -checkout and has libc++ installed in the build directory): - - CFLAGS="-fsanitize=address -isysroot $(xcrun -show-sdk-path)" \ - LDFLAGS=-fsanitize=address CXX=path/to/afl-1.20b/afl-clang++ \ - ./configure.py - AFL_CXX=path/to/clang++ ninja - -Make sure ninja can find the asan runtime: - - DYLD_LIBRARY_PATH=path/to//lib/clang/3.7.0/lib/darwin/ \ - afl-fuzz -i misc/afl-fuzz -o /tmp/afl-fuzz-out ./ninja -n -f @@ diff --git a/README b/README deleted file mode 100644 index a1535ffac8..0000000000 --- a/README +++ /dev/null @@ -1,21 +0,0 @@ -Ninja is a small build system with a focus on speed. -https://ninja-build.org/ - -See the manual -- https://ninja-build.org/manual.html or -doc/manual.asciidoc included in the distribution -- for background -and more details. - -Binaries for Linux, Mac, and Windows are available at - https://github.com/ninja-build/ninja/releases -Run './ninja -h' for Ninja help. - -To build your own binary, on many platforms it should be sufficient to -just run `./configure.py --bootstrap`; for more details see HACKING.md. -(Also read that before making changes to Ninja, as it has advice.) - -Installation is not necessary because the only required file is the -resulting ninja binary. However, to enable features like Bash -completion and Emacs and Vim editing modes, some files in misc/ must be -copied to appropriate locations. - -If you're interested in making changes to Ninja, read HACKING.md first. diff --git a/README.md b/README.md new file mode 100644 index 0000000000..961dddc2fc --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# Ninja + +Ninja is a small build system with a focus on speed. +https://ninja-build.org/ + +See [the manual](https://ninja-build.org/manual.html) or +`doc/manual.asciidoc` included in the distribution for background +and more details. + +Binaries for Linux, Mac and Windows are available on + [GitHub](https://github.com/ninja-build/ninja/releases). +Run `./ninja -h` for Ninja help. + +Installation is not necessary because the only required file is the +resulting ninja binary. However, to enable features like Bash +completion and Emacs and Vim editing modes, some files in misc/ must be +copied to appropriate locations. + +If you're interested in making changes to Ninja, read +[CONTRIBUTING.md](CONTRIBUTING.md) first. + +## Building Ninja itself + +You can either build Ninja via the custom generator script written in Python or +via CMake. For more details see +[the wiki](https://github.com/ninja-build/ninja/wiki). + +### Python + +``` +./configure.py --bootstrap +``` + +This will generate the `ninja` binary and a `build.ninja` file you can now use +to build Ninja with itself. + +If you have a GoogleTest source directory, you can build the tests +by passing its path with `--gtest-source-dir=PATH` option, or the +`GTEST_SOURCE_DIR` environment variable, e.g.: + +``` +./configure.py --bootstrap --gtest-source-dir=/path/to/googletest +./ninja all # build ninja_test and other auxiliary binaries +./ninja_test` # run the unit-test suite. +``` + +Use the CMake build below if you want to use a preinstalled binary +version of the library. + +### CMake + +``` +cmake -Bbuild-cmake +cmake --build build-cmake +``` + +The `ninja` binary will now be inside the `build-cmake` directory (you can +choose any other name you like). + +To run the unit tests: + +``` +./build-cmake/ninja_test +``` + +## Generating documentation + +### Ninja Manual + +You must have `asciidoc` and `xsltproc` in your PATH, then do: + +``` +./configure.py +ninja manual doc/manual.html +``` + +Which will generate `doc/manual.html`. + +To generate the PDF version of the manual, you must have `dblatext` in your PATH then do: + +``` +./configure.py # only if you didn't do it previously. +ninja doc/manual.pdf +``` + +Which will generate `doc/manual.pdf`. + +### Doxygen documentation + +If you have `doxygen` installed, you can build documentation extracted from C++ +declarations and comments to help you navigate the code. Note that Ninja is a standalone +executable, not a library, so there is no public API, all details exposed here are +internal. + +``` +./configure.py # if needed +ninja doxygen +``` + +Then open `doc/doxygen/html/index.html` in a browser to look at it. diff --git a/RELEASING b/RELEASING.md similarity index 50% rename from RELEASING rename to RELEASING.md index da4dbdd0f7..4e3a4bdcc1 100644 --- a/RELEASING +++ b/RELEASING.md @@ -1,33 +1,41 @@ Notes to myself on all the steps to make for a Ninja release. -Push new release branch: -1. Run afl-fuzz for a day or so (see HACKING.md) and run ninja_test +### Push new release branch: +1. Run afl-fuzz for a day or so and run ninja_test 2. Consider sending a heads-up to the ninja-build mailing list first 3. Make sure branches 'master' and 'release' are synced up locally 4. Update src/version.cc with new version (with ".git"), then - git commit -am 'mark this 1.5.0.git' + ``` + git commit -am 'mark this 1.5.0.git' + ``` 5. git checkout release; git merge master 6. Fix version number in src/version.cc (it will likely conflict in the above) 7. Fix version in doc/manual.asciidoc (exists only on release branch) 8. commit, tag, push (don't forget to push --tags) - git commit -am v1.5.0; git push origin release - git tag v1.5.0; git push --tags - # Push the 1.5.0.git change on master too: - git checkout master; git push origin master + ``` + git commit -am v1.5.0; git push origin release + git tag v1.5.0; git push --tags + # Push the 1.5.0.git change on master too: + git checkout master; git push origin master + ``` 9. Construct release notes from prior notes - credits: git shortlog -s --no-merges REV.. -Release on github: -1. https://github.com/blog/1547-release-your-software - Add binaries to https://github.com/ninja-build/ninja/releases + credits: `git shortlog -s --no-merges REV..` -Make announcement on mailing list: + +### Release on GitHub: +1. Go to [Tags](https://github.com/ninja-build/ninja/tags) +2. Open the newly created tag and select "Create release from tag" +3. Create the release which will trigger a build which automatically attaches + the binaries + +### Make announcement on mailing list: 1. copy old mail -Update website: +### Update website: 1. Make sure your ninja checkout is on the v1.5.0 tag 2. Clone https://github.com/ninja-build/ninja-build.github.io 3. In that repo, `./update-docs.sh` 4. Update index.html with newest version and link to release notes -5. git commit -m 'run update-docs.sh, 1.5.0 release' -6. git push origin master +5. `git commit -m 'run update-docs.sh, 1.5.0 release'` +6. `git push origin master` diff --git a/appveyor.yml b/appveyor.yml index 4c64f291d5..505e1423af 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,12 +1,21 @@ version: 1.0.{build} -image: Visual Studio 2017 +image: + - Visual Studio 2017 + - Ubuntu2204 environment: CLICOLOR_FORCE: 1 CHERE_INVOKING: 1 # Tell Bash to inherit the current working directory matrix: - MSYSTEM: MINGW64 - - MSYSTEM: MSVC + - MSYSTEM: LINUX + +matrix: + exclude: + - image: Visual Studio 2017 + MSYSTEM: LINUX + - image: Ubuntu2204 + MSYSTEM: MINGW64 for: - @@ -16,25 +25,16 @@ for: build_script: ps: "C:\\msys64\\usr\\bin\\bash -lc @\"\n pacman -S --quiet --noconfirm --needed re2c 2>&1\n - sed -i 's|cmd /c $ar cqs $out.tmp $in && move /Y $out.tmp $out|$ar crs $out $in|g' configure.py\n ./configure.py --bootstrap --platform mingw 2>&1\n ./ninja all\n - ./ninja_test 2>&1\n ./misc/ninja_syntax_test.py 2>&1\n\"@" - - - matrix: + - matrix: only: - - MSYSTEM: MSVC + - image: Ubuntu2204 build_script: - - cmd: >- - call "C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat" - - python configure.py --bootstrap - - ninja.bootstrap.exe all - - ninja_test - - python misc/ninja_syntax_test.py + - ./configure.py --bootstrap + - ./ninja all + - misc/ninja_syntax_test.py + - misc/output_test.py test: off diff --git a/bootstrap.py b/bootstrap.py deleted file mode 100755 index 56eab64d18..0000000000 --- a/bootstrap.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python -# Copyright 2011 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import print_function - -import subprocess -import sys - -print('DEPRECATED: this script will be deleted.') -print('use "configure.py --bootstrap" instead.') -subprocess.check_call([sys.executable, 'configure.py', '--bootstrap']) diff --git a/configure.py b/configure.py index 850bb98a03..59fc0b8ecf 100755 --- a/configure.py +++ b/configure.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright 2001 Google Inc. All Rights Reserved. # @@ -19,23 +19,24 @@ Projects that use ninja themselves should either write a similar script or use a meta-build system that supports Ninja output.""" -from __future__ import print_function - from optparse import OptionParser import os -import pipes -import string +import shlex import subprocess import sys +from typing import Optional, Union, Dict, List, Any, TYPE_CHECKING sourcedir = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(sourcedir, 'misc')) -import ninja_syntax +if TYPE_CHECKING: + import misc.ninja_syntax as ninja_syntax +else: + import ninja_syntax class Platform(object): """Represents a host/target platform and its specific build attributes.""" - def __init__(self, platform): + def __init__(self, platform: Optional[str]) -> None: self._platform = platform if self._platform is not None: return @@ -60,56 +61,61 @@ def __init__(self, platform): self._platform = 'netbsd' elif self._platform.startswith('aix'): self._platform = 'aix' + elif self._platform.startswith('os400'): + self._platform = 'os400' elif self._platform.startswith('dragonfly'): self._platform = 'dragonfly' @staticmethod - def known_platforms(): + def known_platforms() -> List[str]: return ['linux', 'darwin', 'freebsd', 'openbsd', 'solaris', 'sunos5', 'mingw', 'msvc', 'gnukfreebsd', 'bitrig', 'netbsd', 'aix', 'dragonfly'] - def platform(self): - return self._platform + def platform(self) -> str: + return self._platform # type: ignore # Incompatible return value type - def is_linux(self): + def is_linux(self) -> bool: return self._platform == 'linux' - def is_mingw(self): + def is_mingw(self) -> bool: return self._platform == 'mingw' - def is_msvc(self): + def is_msvc(self) -> bool: return self._platform == 'msvc' - def msvc_needs_fs(self): - popen = subprocess.Popen(['cl', '/nologo', '/?'], + def msvc_needs_fs(self) -> bool: + popen = subprocess.Popen(['cl', '/nologo', '/help'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = popen.communicate() return b'/FS' in out - def is_windows(self): + def is_windows(self) -> bool: return self.is_mingw() or self.is_msvc() - def is_solaris(self): + def is_solaris(self) -> bool: return self._platform == 'solaris' - def is_aix(self): + def is_aix(self) -> bool: return self._platform == 'aix' - def uses_usr_local(self): + def is_os400_pase(self) -> bool: + return self._platform == 'os400' or os.uname().sysname.startswith('OS400') # type: ignore # Module has no attribute "uname" + + def uses_usr_local(self) -> bool: return self._platform in ('freebsd', 'openbsd', 'bitrig', 'dragonfly', 'netbsd') - def supports_ppoll(self): + def supports_ppoll(self) -> bool: return self._platform in ('freebsd', 'linux', 'openbsd', 'bitrig', 'dragonfly') - def supports_ninja_browse(self): + def supports_ninja_browse(self) -> bool: return (not self.is_windows() and not self.is_solaris() and not self.is_aix()) - def can_rebuild_in_place(self): + def can_rebuild_in_place(self) -> bool: return not (self.is_windows() or self.is_aix()) class Bootstrap: @@ -120,37 +126,43 @@ class is used to execute all the commands to build an executable. It also proxies all calls to an underlying ninja_syntax.Writer, to behave like non-bootstrap mode. """ - def __init__(self, writer, verbose=False): + def __init__(self, writer: ninja_syntax.Writer, verbose: bool = False) -> None: self.writer = writer self.verbose = verbose # Map of variable name => expanded variable value. - self.vars = {} + self.vars: Dict[str, str] = {} # Map of rule name => dict of rule attributes. - self.rules = { + self.rules: Dict[str, Dict[str, Any]] = { 'phony': {} } - def comment(self, text): + def comment(self, text: str) -> None: return self.writer.comment(text) - def newline(self): + def newline(self) -> None: return self.writer.newline() - def variable(self, key, val): + def variable(self, key: str, val: str) -> None: # In bootstrap mode, we have no ninja process to catch /showIncludes # output. self.vars[key] = self._expand(val).replace('/showIncludes', '') return self.writer.variable(key, val) - def rule(self, name, **kwargs): + def rule(self, name: str, **kwargs: Any) -> None: self.rules[name] = kwargs return self.writer.rule(name, **kwargs) - def build(self, outputs, rule, inputs=None, **kwargs): + def build( + self, + outputs: Union[str, List[str]], + rule: str, + inputs: Optional[Union[str, List[str]]] = None, + **kwargs: Any + ) -> List[str]: ruleattr = self.rules[rule] cmd = ruleattr.get('command') if cmd is None: # A phony rule, for example. - return + return # type: ignore # Return value expected # Implement just enough of Ninja variable expansion etc. to # make the bootstrap build work. @@ -165,23 +177,23 @@ def build(self, outputs, rule, inputs=None, **kwargs): return self.writer.build(outputs, rule, inputs, **kwargs) - def default(self, paths): + def default(self, paths: Union[str, List[str]]) -> None: return self.writer.default(paths) - def _expand_paths(self, paths): + def _expand_paths(self, paths: Optional[Union[str, List[str]]]) -> str: """Expand $vars in an array of paths, e.g. from a 'build' block.""" paths = ninja_syntax.as_list(paths) return ' '.join(map(self._shell_escape, (map(self._expand, paths)))) - def _expand(self, str, local_vars={}): + def _expand(self, str: str, local_vars: Dict[str, str] = {}) -> str: """Expand $vars in a string.""" return ninja_syntax.expand(str, self.vars, local_vars) - def _shell_escape(self, path): + def _shell_escape(self, path: str) -> str: """Quote paths containing spaces.""" return '"%s"' % path if ' ' in path else path - def _run_command(self, cmdline): + def _run_command(self, cmdline: str) -> None: """Run a subcommand, quietly. Prints the full command on error.""" try: if self.verbose: @@ -211,7 +223,10 @@ def _run_command(self, cmdline): parser.add_option('--profile', metavar='TYPE', choices=profilers, help='enable profiling (' + '/'.join(profilers) + ')',) -parser.add_option('--with-gtest', metavar='PATH', help='ignored') +parser.add_option('--gtest-source-dir', metavar='PATH', + help='Path to GoogleTest source directory. If not provided ' + + 'GTEST_SOURCE_DIR will be probed in the environment. ' + + 'Tests will not be built without a value.') parser.add_option('--with-python', metavar='EXE', help='use EXE as the Python interpreter', default=os.path.basename(sys.executable)) @@ -231,7 +246,7 @@ def _run_command(self, cmdline): BUILD_FILENAME = 'build.ninja' ninja_writer = ninja_syntax.Writer(open(BUILD_FILENAME, 'w')) -n = ninja_writer +n: Union[ninja_syntax.Writer, Bootstrap] = ninja_writer if options.bootstrap: # Make the build directory. @@ -242,7 +257,7 @@ def _run_command(self, cmdline): # Wrap ninja_writer with the Bootstrapper, which also executes the # commands. print('bootstrapping ninja...') - n = Bootstrap(n, verbose=options.verbose) + n = Bootstrap(n, verbose=options.verbose) # type: ignore # Incompatible types in assignment n.comment('This file is used to build ninja itself.') n.comment('It is generated by ' + os.path.basename(__file__) + '.') @@ -259,28 +274,28 @@ def _run_command(self, cmdline): env_keys = set(['CXX', 'AR', 'CFLAGS', 'CXXFLAGS', 'LDFLAGS']) configure_env = dict((k, os.environ[k]) for k in os.environ if k in env_keys) if configure_env: - config_str = ' '.join([k + '=' + pipes.quote(configure_env[k]) + config_str = ' '.join([k + '=' + shlex.quote(configure_env[k]) for k in configure_env]) n.variable('configure_env', config_str + '$ ') n.newline() -CXX = configure_env.get('CXX', 'g++') +CXX = configure_env.get('CXX', 'c++') objext = '.o' if platform.is_msvc(): CXX = 'cl' objext = '.obj' -def src(filename): +def src(filename: str) -> str: return os.path.join('$root', 'src', filename) -def built(filename): +def built(filename: str) -> str: return os.path.join('$builddir', filename) -def doc(filename): +def doc(filename: str) -> str: return os.path.join('$root', 'doc', filename) -def cc(name, **kwargs): +def cc(name: str, **kwargs: Any) -> List[str]: return n.build(built(name + objext), 'cxx', src(name + '.c'), **kwargs) -def cxx(name, **kwargs): +def cxx(name: str, **kwargs: Any) -> List[str]: return n.build(built(name + objext), 'cxx', src(name + '.cc'), **kwargs) -def binary(name): +def binary(name: str) -> str: if platform.is_windows(): exe = name + '.exe' n.build(name, 'phony', exe) @@ -300,14 +315,26 @@ def binary(name): else: n.variable('ar', configure_env.get('AR', 'ar')) +def search_system_path(file_name: str) -> Optional[str]: # type: ignore # Missing return statement + """Find a file in the system path.""" + for dir in os.environ['path'].split(';'): + path = os.path.join(dir, file_name) + if os.path.exists(path): + return path + +# Note that build settings are separately specified in CMakeLists.txt and +# these lists should be kept in sync. if platform.is_msvc(): + if not search_system_path('cl.exe'): + raise Exception('cl.exe not found. Run again from the Developer Command Prompt for VS') cflags = ['/showIncludes', '/nologo', # Don't print startup banner. + '/utf-8', '/Zi', # Create pdb with debug info. '/W4', # Highest warning level. '/WX', # Warnings as errors. '/wd4530', '/wd4100', '/wd4706', '/wd4244', - '/wd4512', '/wd4800', '/wd4702', '/wd4819', + '/wd4512', '/wd4800', '/wd4702', # Disable warnings about constant conditional expressions. '/wd4127', # Disable warnings about passing "this" during initialization. @@ -315,6 +342,7 @@ def binary(name): # Disable warnings about ignored typedef in DbgHelp.h '/wd4091', '/GR-', # Disable RTTI. + '/Zc:__cplusplus', # Disable size_t -> int truncation warning. # We never have strings or arrays larger than 2**31. '/wd4267', @@ -334,6 +362,7 @@ def binary(name): '-Wno-unused-parameter', '-fno-rtti', '-fno-exceptions', + '-std=c++14', '-fvisibility=hidden', '-pipe', '-DNINJA_PYTHON="%s"' % options.with_python] if options.debug: @@ -351,7 +380,7 @@ def binary(name): except: pass if platform.is_mingw(): - cflags += ['-D_WIN32_WINNT=0x0501'] + cflags += ['-D_WIN32_WINNT=0x0601', '-D__USE_MINGW_ANSI_STDIO=1'] ldflags = ['-L$builddir'] if platform.uses_usr_local(): cflags.append('-I/usr/local/include') @@ -389,7 +418,7 @@ def binary(name): # Search for generated headers relative to build dir. cflags.append('-I.') -def shell_escape(str): +def shell_escape(str: str) -> str: """Escape str such that it's interpreted as a single argument by the shell.""" @@ -410,6 +439,7 @@ def shell_escape(str): if 'LDFLAGS' in configure_env: ldflags.append(configure_env['LDFLAGS']) n.variable('ldflags', ' '.join(shell_escape(flag) for flag in ldflags)) + n.newline() if platform.is_msvc(): @@ -432,7 +462,7 @@ def shell_escape(str): description='LIB $out') elif host.is_mingw(): n.rule('ar', - command='cmd /c $ar cqs $out.tmp $in && move /Y $out.tmp $out', + command='$ar crs $out $in', description='AR $out') else: n.rule('ar', @@ -466,48 +496,63 @@ def shell_escape(str): n.newline() n.comment('the depfile parser and ninja lexers are generated using re2c.') -def has_re2c(): +def has_re2c() -> bool: try: proc = subprocess.Popen(['re2c', '-V'], stdout=subprocess.PIPE) - return int(proc.communicate()[0], 10) >= 1103 + return int(proc.communicate()[0], 10) >= 1503 except OSError: return False if has_re2c(): n.rule('re2c', - command='re2c -b -i --no-generation-date -o $out $in', + command='re2c -b -i --no-generation-date --no-version -o $out $in', description='RE2C $out') # Generate the .cc files in the source directory so we can check them in. n.build(src('depfile_parser.cc'), 're2c', src('depfile_parser.in.cc')) n.build(src('lexer.cc'), 're2c', src('lexer.in.cc')) else: - print("warning: A compatible version of re2c (>= 0.11.3) was not found; " + print("warning: A compatible version of re2c (>= 0.15.3) was not found; " "changes to src/*.in.cc will not affect your build.") n.newline() -n.comment('Core source files all build into ninja library.') cxxvariables = [] if platform.is_msvc(): cxxvariables = [('pdb', 'ninja.pdb')] + +n.comment('Generate a library for `ninja-re2c`.') +re2c_objs = [] +for name in ['depfile_parser', 'lexer']: + re2c_objs += cxx(name, variables=cxxvariables) +if platform.is_msvc(): + n.build(built('ninja-re2c.lib'), 'ar', re2c_objs) +else: + n.build(built('libninja-re2c.a'), 'ar', re2c_objs) +n.newline() + +n.comment('Core source files all build into ninja library.') +objs.extend(re2c_objs) for name in ['build', 'build_log', 'clean', 'clparser', 'debug_flags', - 'depfile_parser', 'deps_log', 'disk_interface', 'dyndep', 'dyndep_parser', 'edit_distance', + 'elide_middle', 'eval_env', 'graph', 'graphviz', - 'lexer', + 'json', 'line_printer', 'manifest_parser', 'metrics', + 'missing_deps', 'parser', + 'real_command_runner', 'state', + 'status_printer', 'string_piece_util', 'util', 'version']: @@ -536,7 +581,7 @@ def has_re2c(): else: libs.append('-lninja') -if platform.is_aix(): +if platform.is_aix() and not platform.is_os400_pase(): libs.append('-lperfstat') all_targets = [] @@ -554,45 +599,95 @@ def has_re2c(): # build.ninja file. n = ninja_writer -n.comment('Tests all build into ninja_test executable.') +# Build the ninja_test executable only if the GTest source directory +# is provided explicitly. Either from the environment with GTEST_SOURCE_DIR +# or with the --gtest-source-dir command-line option. +# +# Do not try to look for an installed binary version, and link against it +# because doing so properly is platform-specific (use the CMake build for +# this). +if options.gtest_source_dir: + gtest_src_dir = options.gtest_source_dir +else: + gtest_src_dir = os.environ.get('GTEST_SOURCE_DIR') -objs = [] -if platform.is_msvc(): - cxxvariables = [('pdb', 'ninja_test.pdb')] - -for name in ['build_log_test', - 'build_test', - 'clean_test', - 'clparser_test', - 'depfile_parser_test', - 'deps_log_test', - 'dyndep_parser_test', - 'disk_interface_test', - 'edit_distance_test', - 'graph_test', - 'lexer_test', - 'manifest_parser_test', - 'ninja_test', - 'state_test', - 'string_piece_util_test', - 'subprocess_test', - 'test', - 'util_test']: - objs += cxx(name, variables=cxxvariables) -if platform.is_windows(): - for name in ['includes_normalize_test', 'msvc_helper_test']: - objs += cxx(name, variables=cxxvariables) +if gtest_src_dir: + # Verify GoogleTest source directory, and add its include directory + # to the global include search path (even for non-test sources) to + # keep the build plan generation simple. + gtest_all_cc = os.path.join(gtest_src_dir, 'googletest', 'src', 'gtest-all.cc') + if not os.path.exists(gtest_all_cc): + print('ERROR: Missing GoogleTest source file: %s' % gtest_all_cc) + sys.exit(1) -ninja_test = n.build(binary('ninja_test'), 'link', objs, implicit=ninja_lib, - variables=[('libs', libs)]) -n.newline() -all_targets += ninja_test + n.comment('Tests all build into ninja_test executable.') + + # Test-specific version of cflags, must include the GoogleTest + # include directory. + test_cflags = cflags.copy() + test_cflags.append('-I' + os.path.join(gtest_src_dir, 'googletest', 'include')) + test_variables = [('cflags', test_cflags)] + if platform.is_msvc(): + test_variables += [('pdb', 'ninja_test.pdb')] + + test_names = [ + 'build_log_test', + 'build_test', + 'clean_test', + 'clparser_test', + 'depfile_parser_test', + 'deps_log_test', + 'disk_interface_test', + 'dyndep_parser_test', + 'edit_distance_test', + 'elide_middle_test', + 'explanations_test', + 'graph_test', + 'json_test', + 'lexer_test', + 'manifest_parser_test', + 'ninja_test', + 'state_test', + 'string_piece_util_test', + 'subprocess_test', + 'test', + 'util_test', + ] + if platform.is_windows(): + test_names += [ + 'includes_normalize_test', + 'msvc_helper_test', + ] + + objs = [] + for name in test_names: + objs += cxx(name, variables=test_variables) + + # Build GTest as a monolithic source file. + # This requires one extra include search path, so replace the + # value of 'cflags' in our list. + gtest_all_variables = test_variables[1:] + [ + ('cflags', test_cflags + ['-I' + os.path.join(gtest_src_dir, 'googletest') ]), + ] + # Do not use cxx() directly to ensure the object file is under $builddir. + objs += n.build(built('gtest_all' + objext), 'cxx', gtest_all_cc, variables=gtest_all_variables) + + ninja_test = n.build(binary('ninja_test'), 'link', objs, implicit=ninja_lib, + variables=[('libs', libs)]) + n.newline() + all_targets += ninja_test n.comment('Ancillary executables.') +if platform.is_aix() and '-maix64' not in ldflags: + # Both hash_collision_bench and manifest_parser_perftest require more + # memory than will fit in the standard 32-bit AIX shared stack/heap (256M) + libs.append('-Wl,-bmaxdata:0x80000000') + for name in ['build_log_perftest', 'canon_perftest', + 'elide_middle_perftest', 'depfile_parser_perftest', 'hash_collision_bench', 'manifest_parser_perftest', @@ -644,7 +739,7 @@ def has_re2c(): command='$doxygen_mainpage_generator $in > $out', description='DOXYGEN_MAINPAGE $out') mainpage = n.build(built('doxygen_mainpage'), 'doxygen_mainpage', - ['README', 'COPYING'], + ['README.md', 'COPYING'], implicit=['$doxygen_mainpage_generator']) n.build('doxygen', 'doxygen', doc('doxygen.config'), implicit=mainpage) @@ -674,7 +769,7 @@ def has_re2c(): n.build('all', 'phony', all_targets) -n.close() +n.close() # type: ignore # Item "Bootstrap" of "Writer | Bootstrap" has no attribute "close" print('wrote %s.' % BUILD_FILENAME) if options.bootstrap: diff --git a/doc/manual.asciidoc b/doc/manual.asciidoc index 7f3ab8afd1..1e9ede9891 100644 --- a/doc/manual.asciidoc +++ b/doc/manual.asciidoc @@ -24,7 +24,7 @@ Where other build systems are high-level languages, Ninja aims to be an assembler. Build systems get slow when they need to make decisions. When you are -in a edit-compile cycle you want it to be as fast as possible -- you +in an edit-compile cycle you want it to be as fast as possible -- you want the build system to do the minimum work necessary to figure out what needs to be built immediately. @@ -197,14 +197,18 @@ Several placeholders are available: `%s`:: The number of started edges. `%t`:: The total number of edges that must be run to complete the build. -`%p`:: The percentage of started edges. +`%p`:: The percentage of finished edges. `%r`:: The number of currently running edges. `%u`:: The number of remaining edges to start. `%f`:: The number of finished edges. `%o`:: Overall rate of finished edges per second `%c`:: Current rate of finished edges per second (average over builds specified by `-j` or its default) -`%e`:: Elapsed time in seconds. _(Available since Ninja 1.2.)_ +`%e`:: Elapsed time in seconds. _(Available since Ninja 1.2.)_ +`%E`:: Remaining time (ETA) in seconds. _(Available since Ninja 1.12.)_ +`%w`:: Elapsed time in [h:]mm:ss format. _(Available since Ninja 1.12.)_ +`%W`:: Remaining time (ETA) in [h:]mm:ss format. _(Available since Ninja 1.12.)_ +`%P`:: The percentage (in ppp% format) of time elapsed out of predicted total runtime. _(Available since Ninja 1.12.)_ `%%`:: A plain `%` character. The default progress status is `"[%f/%t] "` (note the trailing space @@ -222,14 +226,14 @@ found useful during Ninja's development. The current tools are: `browse`:: browse the dependency graph in a web browser. Clicking a file focuses the view on that file, showing inputs and outputs. This -feature requires a Python installation. By default port 8000 is used +feature requires a Python installation. By default, port 8000 is used and a web browser will be opened. This can be changed as follows: + ---- ninja -t browse --port=8000 --no-browser mytarget ---- + -`graph`:: output a file in the syntax used by `graphviz`, a automatic +`graph`:: output a file in the syntax used by `graphviz`, an automatic graph layout tool. Use it like: + ---- @@ -257,7 +261,52 @@ than the _depth_ mode. executed in order, may be used to rebuild those targets, assuming that all output files are out of date. -`clean`:: remove built files. By default it removes all built files +`inputs`:: given a list of targets, print a list of all inputs used to +rebuild those targets. +_Available since Ninja 1.11._ + +`multi-inputs`:: print one or more sets of inputs required to build targets. +Each line will consist of a target, a delimiter, an input and a terminator character. +The list produced by the tool can be helpful if one would like to know which targets +that are affected by a certain input. ++ +The output will be a series of lines with the following elements: ++ +---- + +---- ++ +The default `` is a single TAB character. +The delimiter can be modified to any string using the `--delimiter` argument. ++ +The default `` is a line terminator (i.e. `\n` on Posix and `\r\n` on Windows). +The terminator can be changed to `\0` by using the `--print0` argument. ++ +---- +---- ++ +Example usage of the `multi-inputs` tool: ++ +---- +ninja -t multi-inputs target1 target2 target3 +---- ++ +Example of produced output from the `multi-inputs` tool: ++ +---- +target1 file1.c +target2 file1.c +target2 file2.c +target3 file1.c +target3 file2.c +target3 file3.c +---- ++ +_Note that a given input may appear for several targets if it is used by more +than one targets._ +_Available since Ninja 1.13._ + +`clean`:: remove built files. By default, it removes all built files except for those created by the generator. Adding the `-g` flag also removes built files created by the generator (see <>). Additional arguments are @@ -271,6 +320,9 @@ Files created but not referenced in the graph are not removed. This tool takes in account the +-v+ and the +-n+ options (note that +-n+ implies +-v+). +`cleandead`:: remove files produced by previous builds that are no longer in the +build file. _Available since Ninja 1.10._ + `compdb`:: given a list of rules, each of which is expected to be a C family language compiler rule whose first input is the name of the source file, prints on standard output a compilation database in the @@ -278,14 +330,83 @@ http://clang.llvm.org/docs/JSONCompilationDatabase.html[JSON format] expected by the Clang tooling interface. _Available since Ninja 1.2._ +`compdb-targets`:: like `compdb`, but takes a list of targets instead of rules, +and expects at least one target. The resulting compilation database contains +all commands required to build the indicated targets, and _only_ those +commands. + `deps`:: show all dependencies stored in the `.ninja_deps` file. When given a target, show just the target's dependencies. _Available since Ninja 1.4._ +`missingdeps`:: given a list of targets, look for targets that depend on +a generated file, but do not have a properly (possibly transitive) dependency +on the generator. Such targets may cause build flakiness on clean builds. ++ +The broken targets can be found assuming deps log / depfile dependency +information is correct. Any target that depends on a generated file (output +of a generator-target) implicitly, but does not have an explicit or order-only +dependency path to the generator-target, is considered broken. ++ +The tool's findings can be verified by trying to build the listed targets in +a clean outdir without building any other targets. The build should fail for +each of them with a missing include error or equivalent pointing to the +generated file. +_Available since Ninja 1.11._ + `recompact`:: recompact the `.ninja_deps` file. _Available since Ninja 1.4._ -`rules`:: output the list of all rules (eventually with their description -if they have one). It can be used to know which rule name to pass to -+ninja -t targets rule _name_+ or +ninja -t compdb+. +`restat`:: updates all recorded file modification timestamps in the `.ninja_log` +file. _Available since Ninja 1.10._ + +`rules`:: output the list of all rules. It can be used to know which rule name +to pass to +ninja -t targets rule _name_+ or +ninja -t compdb+. Adding the `-d` +flag also prints the description of the rules. + +`msvc`:: Available on Windows hosts only. +Helper tool to invoke the `cl.exe` compiler with a pre-defined set of +environment variables, as in: ++ +---- +ninja -t msvc -e ENVFILE -- cl.exe +---- ++ +Where `ENVFILE` is a binary file that contains an environment block suitable +for CreateProcessA() on Windows (i.e. a series of zero-terminated strings that +look like NAME=VALUE, followed by an extra zero terminator). Note that this uses +the local codepage encoding. ++ +This tool also supports a deprecated way of parsing the compiler's output when +the `/showIncludes` flag is used, and generating a GCC-compatible depfile from it: ++ +---- +ninja -t msvc -o DEPFILE [-p STRING] -- cl.exe /showIncludes +---- ++ +When using this option, `-p STRING` can be used to pass the localized line prefix +that `cl.exe` uses to output dependency information. For English-speaking regions +this is `"Note: including file: "` without the double quotes, but will be different +for other regions. ++ +Note that Ninja supports this natively now, with the use of `deps = msvc` and +`msvc_deps_prefix` in Ninja files. Native support also avoids launching an extra +tool process each time the compiler must be called, which can speed up builds +noticeably on Windows. + +`wincodepage`:: Available on Windows hosts (_since Ninja 1.11_). +Prints the Windows code page whose encoding is expected in the build file. +The output has the form: ++ +---- +Build file encoding: +---- ++ +Additional lines may be added in future versions of Ninja. ++ +The `` is one of: + +`UTF-8`::: Encode as UTF-8. + +`ANSI`::: Encode to the system-wide ANSI code page. Writing your own Ninja files ---------------------------- @@ -446,12 +567,25 @@ nothing, but phony rules are handled specially in that they aren't printed when run, logged (see below), nor do they contribute to the command count printed as part of the build process. +When a `phony` target is used as an input to another build rule, the +other build rule will, semantically, consider the inputs of the +`phony` rule as its own. Therefore, `phony` rules can be used to group +inputs, e.g. header files. + `phony` can also be used to create dummy targets for files which may not exist at build time. If a phony build statement is written without any dependencies, the target will be considered out of date if it does not exist. Without a phony build statement, Ninja will report an error if the file does not exist and is required by the build. +To create a rule that never rebuilds, use a build rule without any input: +---------------- +rule touch + command = touch $out +build file_that_always_exists.dummy: touch +build dummy_target_to_follow_a_pattern: phony file_that_always_exists.dummy +---------------- + Default target statements ~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -561,10 +695,10 @@ Use it like in the following example: ---- rule cc depfile = $out.d - command = gcc -MMD -MF $out.d [other gcc flags here] + command = gcc -MD -MF $out.d [other gcc flags here] ---- -The `-MMD` flag to `gcc` tells it to output header dependencies, and +The `-MD` flag to `gcc` tells it to output header dependencies, and the `-MF` flag tells it where to write them. deps @@ -589,14 +723,14 @@ Ninja supports this processing in two forms. as a temporary). 2. `deps = msvc` specifies that the tool outputs header dependencies - in the form produced by Visual Studio's compiler's + in the form produced by the Visual Studio compiler's http://msdn.microsoft.com/en-us/library/hdkef6tk(v=vs.90).aspx[`/showIncludes` flag]. Briefly, this means the tool outputs specially-formatted lines to its stdout. Ninja then filters these lines from the displayed output. No `depfile` attribute is necessary, but the localized string - in front of the the header file path. For instance + in front of the header file path should be globally defined. For instance, `msvc_deps_prefix = Note: including file:` - for a English Visual Studio (the default). Should be globally defined. + for an English Visual Studio (the default). + ---- msvc_deps_prefix = Note: including file: @@ -698,6 +832,8 @@ A file is a series of declarations. A declaration can be one of: Order-only dependencies may be tacked on the end with +|| _dependency1_ _dependency2_+. (See <>.) + Validations may be tacked on the end with +|@ _validation1_ _validation2_+. + (See <>.) + Implicit outputs _(available since Ninja 1.7)_ may be added before the `:` with +| _output1_ _output2_+ and do not appear in `$out`. @@ -877,15 +1013,15 @@ Fundamentally, command lines behave differently on Unixes and Windows. On Unixes, commands are arrays of arguments. The Ninja `command` variable is passed directly to `sh -c`, which is then responsible for -interpreting that string into an argv array. Therefore the quoting +interpreting that string into an argv array. Therefore, the quoting rules are those of the shell, and you can use all the normal shell operators, like `&&` to chain multiple commands, or `VAR=value cmd` to set environment variables. On Windows, commands are strings, so Ninja passes the `command` string directly to `CreateProcess`. (In the common case of simply executing -a compiler this means there is less overhead.) Consequently the -quoting rules are deterimined by the called program, which on Windows +a compiler this means there is less overhead.) Consequently, the +quoting rules are determined by the called program, which on Windows are usually provided by the C library. If you need shell interpretation of the command (such as the use of `&&` to chain multiple commands), make the command execute the Windows shell by @@ -921,7 +1057,7 @@ There are three types of build dependencies which are subtly different. 1. _Explicit dependencies_, as listed in a build line. These are available as the `$in` variable in the rule. Changes in these files - cause the output to be rebuilt; if these file are missing and + cause the output to be rebuilt; if these files are missing and Ninja doesn't know how to build them, the build is aborted. + This is the standard form of dependency to be used e.g. for the @@ -935,8 +1071,9 @@ source file of a compile command. + This is for expressing dependencies that don't show up on the command line of the command; for example, for a rule that runs a -script, the script itself should be an implicit dependency, as -changes to the script should cause the output to rebuild. +script that reads a hardcoded file, the hardcoded file should +be an implicit dependency, as changes to the file should cause +the output to rebuild, even though it doesn't show up in the arguments. + Note that dependencies as loaded through depfiles have slightly different semantics, as described in the <>. @@ -955,6 +1092,34 @@ express the implicit dependency.) File paths are compared as is, which means that an absolute path and a relative path, pointing to the same file, are considered different by Ninja. +[[validations]] +Validations +~~~~~~~~~~~ + +_Available since Ninja 1.11._ + +Validations listed on the build line cause the specified files to be +added to the top level of the build graph (as if they were specified +on the Ninja command line) whenever the build line is a transitive +dependency of one of the targets specified on the command line or a +default target. + +Validations are added to the build graph regardless of whether the output +files of the build statement are dirty are not, and the dirty state of +the build statement that outputs the file being used as a validation +has no effect on the dirty state of the build statement that requested it. + +A build edge can list another build edge as a validation even if the second +edge depends on the first. + +Validations are designed to handle rules that perform error checking but +don't produce any artifacts needed by the build, for example, static +analysis tools. Marking the static analysis rule as an implicit input +of the main build rule of the source files or of the rules that depend +on the main build rule would slow down the critical path of the build, +but using a validation would allow the build to proceed in parallel with +the static analysis rule once the main build rule is complete. + Variable expansion ~~~~~~~~~~~~~~~~~~ diff --git a/doc/style.css b/doc/style.css index 9976c03ac3..2be09de8b1 100644 --- a/doc/style.css +++ b/doc/style.css @@ -1,15 +1,22 @@ +:root { + color-scheme: light dark; +} + body { margin: 5ex 10ex; max-width: 80ex; line-height: 1.5; font-family: sans-serif; } + h1, h2, h3 { font-weight: normal; } + pre, code { font-family: x, monospace; } + pre { padding: 1ex; background: #eee; @@ -17,13 +24,45 @@ pre { min-width: 0; font-size: 90%; } +@media (prefers-color-scheme: dark) { + pre { + background: #333; + border: solid 1px #444; + } +} + code { color: #007; } +@media (prefers-color-scheme: dark) { + code { + color: #a7cec8; + } +} + div.chapter { margin-top: 4em; border-top: solid 2px black; } +@media (prefers-color-scheme: dark) { + div.chapter { + border-top: solid 2px white; + } +} + p { margin-top: 0; } + +/* The following applies to the left column of a [horizontal] labeled list: */ +table.horizontal > tbody > tr > td:nth-child(1) { + + /* prevent the insertion of a line-break in the middle of a label: */ + white-space: nowrap; + + /* insert a little horizontal padding between the two columns: */ + padding-right: 1.5em; + + /* right-justify labels: */ + text-align: end; +} diff --git a/misc/ci.py b/misc/ci.py index 17cbf14698..20a4415f8a 100755 --- a/misc/ci.py +++ b/misc/ci.py @@ -5,22 +5,27 @@ ignores = [ '.git/', 'misc/afl-fuzz-tokens/', - 'ninja_deps', 'src/depfile_parser.cc', 'src/lexer.cc', ] error_count = 0 -def error(path, msg): +def error(path: str, msg: str) -> None: global error_count error_count += 1 print('\x1b[1;31m{}\x1b[0;31m{}\x1b[0m'.format(path, msg)) +try: + import git + repo = git.Repo('.') +except: + repo = None + for root, directory, filenames in os.walk('.'): for filename in filenames: path = os.path.join(root, filename)[2:] - if any([path.startswith(x) for x in ignores]): + if any([path.startswith(x) for x in ignores]) or (repo is not None and repo.ignored(path)): continue with open(path, 'rb') as file: line_nr = 1 diff --git a/misc/manifest_fuzzer.cc b/misc/manifest_fuzzer.cc new file mode 100644 index 0000000000..085840a388 --- /dev/null +++ b/misc/manifest_fuzzer.cc @@ -0,0 +1,41 @@ +// Copyright 2020 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "stdint.h" +#include +#include "disk_interface.h" +#include "state.h" +#include "manifest_parser.h" +#include + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + char build_file[256]; + sprintf(build_file, "/tmp/build.ninja"); + FILE *fp = fopen(build_file, "wb"); + if (!fp) + return 0; + fwrite(data, size, 1, fp); + fclose(fp); + + std::string err; + RealDiskInterface disk_interface; + State state; + ManifestParser parser(&state, &disk_interface); + + parser.Load("/tmp/build.ninja", &err); + + std::__fs::filesystem::remove_all("/tmp/build.ninja"); + return 0; +} diff --git a/misc/measure.py b/misc/measure.py index 8ce95e696b..e80880471b 100755 --- a/misc/measure.py +++ b/misc/measure.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2011 Google Inc. All Rights Reserved. # @@ -17,15 +17,14 @@ """measure the runtime of a command by repeatedly running it. """ -from __future__ import print_function - import time import subprocess import sys +from typing import Union, List devnull = open('/dev/null', 'w') -def run(cmd, repeat=10): +def run(cmd: Union[str, List[str]], repeat: int = 10) -> None: print('sampling:', end=' ') sys.stdout.flush() diff --git a/misc/ninja-mode.el b/misc/ninja-mode.el deleted file mode 100644 index 8b975d5156..0000000000 --- a/misc/ninja-mode.el +++ /dev/null @@ -1,85 +0,0 @@ -;;; ninja-mode.el --- Major mode for editing .ninja files -*- lexical-binding: t -*- - -;; Package-Requires: ((emacs "24")) - -;; Copyright 2011 Google Inc. All Rights Reserved. -;; -;; Licensed under the Apache License, Version 2.0 (the "License"); -;; you may not use this file except in compliance with the License. -;; You may obtain a copy of the License at -;; -;; http://www.apache.org/licenses/LICENSE-2.0 -;; -;; Unless required by applicable law or agreed to in writing, software -;; distributed under the License is distributed on an "AS IS" BASIS, -;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -;; See the License for the specific language governing permissions and -;; limitations under the License. - -;;; Commentary: - -;; Simple emacs mode for editing .ninja files. -;; Just some syntax highlighting for now. - -;;; Code: - -(defvar ninja-keywords - `((,(concat "^" (regexp-opt '("rule" "build" "subninja" "include" - "pool" "default") - 'words)) - . font-lock-keyword-face) - ("\\([[:alnum:]_]+\\) =" 1 font-lock-variable-name-face) - ;; Variable expansion. - ("$[[:alnum:]_]+" . font-lock-variable-name-face) - ("${[[:alnum:]._]+}" . font-lock-variable-name-face) - ;; Rule names - ("rule +\\([[:alnum:]_.-]+\\)" 1 font-lock-function-name-face) - ;; Build Statement - highlight the rule used, - ;; allow for escaped $,: in outputs. - ("build +\\(?:[^:$\n]\\|$[:$]\\)+ *: *\\([[:alnum:]_.-]+\\)" - 1 font-lock-function-name-face))) - -(defvar ninja-mode-syntax-table - (let ((table (make-syntax-table))) - (modify-syntax-entry ?\" "." table) - table) - "Syntax table used in `ninja-mode'.") - -(defun ninja-syntax-propertize (start end) - (save-match-data - (goto-char start) - (while (search-forward "#" end t) - (let ((match-pos (match-beginning 0))) - (when (and - ;; Is it the first non-white character on the line? - (eq match-pos (save-excursion (back-to-indentation) (point))) - (save-excursion - (goto-char (line-end-position 0)) - (or - ;; If we're continuing the previous line, it's not a - ;; comment. - (not (eq ?$ (char-before))) - ;; Except if the previous line is a comment as well, as the - ;; continuation dollar is ignored then. - (nth 4 (syntax-ppss))))) - (put-text-property match-pos (1+ match-pos) 'syntax-table '(11)) - (let ((line-end (line-end-position))) - ;; Avoid putting properties past the end of the buffer. - ;; Otherwise we get an `args-out-of-range' error. - (unless (= line-end (1+ (buffer-size))) - (put-text-property line-end (1+ line-end) 'syntax-table '(12))))))))) - -;;;###autoload -(define-derived-mode ninja-mode prog-mode "ninja" - (set (make-local-variable 'comment-start) "#") - (set (make-local-variable 'parse-sexp-lookup-properties) t) - (set (make-local-variable 'syntax-propertize-function) #'ninja-syntax-propertize) - (setq font-lock-defaults '(ninja-keywords))) - -;; Run ninja-mode for files ending in .ninja. -;;;###autoload -(add-to-list 'auto-mode-alist '("\\.ninja$" . ninja-mode)) - -(provide 'ninja-mode) - -;;; ninja-mode.el ends here diff --git a/misc/ninja_syntax.py b/misc/ninja_syntax.py index ebe6490d8d..2aa8456e9d 100644 --- a/misc/ninja_syntax.py +++ b/misc/ninja_syntax.py @@ -1,5 +1,19 @@ #!/usr/bin/python +# Copyright 2011 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Python module for generating .ninja files. Note that this is emphatically not a required piece of Ninja; it's @@ -9,37 +23,54 @@ import re import textwrap +from io import TextIOWrapper +from typing import Dict, List, Match, Optional, Tuple, Union -def escape_path(word): +def escape_path(word: str) -> str: return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:') class Writer(object): - def __init__(self, output, width=78): + def __init__(self, output: TextIOWrapper, width: int = 78) -> None: self.output = output self.width = width - def newline(self): + def newline(self) -> None: self.output.write('\n') - def comment(self, text): + def comment(self, text: str) -> None: for line in textwrap.wrap(text, self.width - 2, break_long_words=False, break_on_hyphens=False): self.output.write('# ' + line + '\n') - def variable(self, key, value, indent=0): + def variable( + self, + key: str, + value: Optional[Union[bool, int, float, str, List[str]]], + indent: int = 0, + ) -> None: if value is None: return if isinstance(value, list): value = ' '.join(filter(None, value)) # Filter out empty strings. self._line('%s = %s' % (key, value), indent) - def pool(self, name, depth): + def pool(self, name: str, depth: int) -> None: self._line('pool %s' % name) self.variable('depth', depth, indent=1) - def rule(self, name, command, description=None, depfile=None, - generator=False, pool=None, restat=False, rspfile=None, - rspfile_content=None, deps=None): + def rule( + self, + name: str, + command: str, + description: Optional[str] = None, + depfile: Optional[str] = None, + generator: bool = False, + pool: Optional[str] = None, + restat: bool = False, + rspfile: Optional[str] = None, + rspfile_content: Optional[str] = None, + deps: Optional[Union[str, List[str]]] = None, + ) -> None: self._line('rule %s' % name) self.variable('command', command, indent=1) if description: @@ -59,8 +90,23 @@ def rule(self, name, command, description=None, depfile=None, if deps: self.variable('deps', deps, indent=1) - def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, - variables=None, implicit_outputs=None, pool=None): + def build( + self, + outputs: Union[str, List[str]], + rule: str, + inputs: Optional[Union[str, List[str]]] = None, + implicit: Optional[Union[str, List[str]]] = None, + order_only: Optional[Union[str, List[str]]] = None, + variables: Optional[ + Union[ + List[Tuple[str, Optional[Union[str, List[str]]]]], + Dict[str, Optional[Union[str, List[str]]]], + ] + ] = None, + implicit_outputs: Optional[Union[str, List[str]]] = None, + pool: Optional[str] = None, + dyndep: Optional[str] = None, + ) -> List[str]: outputs = as_list(outputs) out_outputs = [escape_path(x) for x in outputs] all_inputs = [escape_path(x) for x in as_list(inputs)] @@ -83,6 +129,8 @@ def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, ' '.join([rule] + all_inputs))) if pool is not None: self._line(' pool = %s' % pool) + if dyndep is not None: + self._line(' dyndep = %s' % dyndep) if variables: if isinstance(variables, dict): @@ -95,16 +143,16 @@ def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, return outputs - def include(self, path): + def include(self, path: str) -> None: self._line('include %s' % path) - def subninja(self, path): + def subninja(self, path: str) -> None: self._line('subninja %s' % path) - def default(self, paths): + def default(self, paths: Union[str, List[str]]) -> None: self._line('default %s' % ' '.join(as_list(paths))) - def _count_dollars_before_index(self, s, i): + def _count_dollars_before_index(self, s: str, i: int) -> int: """Returns the number of '$' characters right in front of s[i].""" dollar_count = 0 dollar_index = i - 1 @@ -113,7 +161,7 @@ def _count_dollars_before_index(self, s, i): dollar_index -= 1 return dollar_count - def _line(self, text, indent=0): + def _line(self, text: str, indent: int = 0) -> None: """Write 'text' word-wrapped at self.width characters.""" leading_space = ' ' * indent while len(leading_space) + len(text) > self.width: @@ -149,11 +197,11 @@ def _line(self, text, indent=0): self.output.write(leading_space + text + '\n') - def close(self): + def close(self) -> None: self.output.close() -def as_list(input): +def as_list(input: Optional[Union[str, List[str]]]) -> List[str]: if input is None: return [] if isinstance(input, list): @@ -161,7 +209,7 @@ def as_list(input): return [input] -def escape(string): +def escape(string: str) -> str: """Escape a string such that it can be embedded into a Ninja file without further interpretation.""" assert '\n' not in string, 'Ninja syntax does not allow newlines' @@ -169,13 +217,13 @@ def escape(string): return string.replace('$', '$$') -def expand(string, vars, local_vars={}): +def expand(string: str, vars: Dict[str, str], local_vars: Dict[str, str] = {}) -> str: """Expand a string containing $vars as Ninja would. Note: doesn't handle the full Ninja variable syntax, but it's enough to make configure.py's use of it work. """ - def exp(m): + def exp(m: Match[str]) -> str: var = m.group(1) if var == '$': return '$' diff --git a/misc/ninja_syntax_test.py b/misc/ninja_syntax_test.py index 90ff9c6bdb..34122492fe 100755 --- a/misc/ninja_syntax_test.py +++ b/misc/ninja_syntax_test.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2011 Google Inc. All Rights Reserved. # @@ -15,6 +15,7 @@ # limitations under the License. import unittest +from typing import Dict try: from StringIO import StringIO @@ -28,16 +29,16 @@ INDENT = ' ' class TestLineWordWrap(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: self.out = StringIO() self.n = ninja_syntax.Writer(self.out, width=8) - def test_single_long_word(self): + def test_single_long_word(self) -> None: # We shouldn't wrap a single long word. self.n._line(LONGWORD) self.assertEqual(LONGWORD + '\n', self.out.getvalue()) - def test_few_long_words(self): + def test_few_long_words(self) -> None: # We should wrap a line where the second word is overlong. self.n._line(' '.join(['x', LONGWORD, 'y'])) self.assertEqual(' $\n'.join(['x', @@ -45,13 +46,13 @@ def test_few_long_words(self): INDENT + 'y']) + '\n', self.out.getvalue()) - def test_comment_wrap(self): + def test_comment_wrap(self) -> None: # Filenames should not be wrapped self.n.comment('Hello /usr/local/build-tools/bin') self.assertEqual('# Hello\n# /usr/local/build-tools/bin\n', self.out.getvalue()) - def test_short_words_indented(self): + def test_short_words_indented(self) -> None: # Test that indent is taking into account when breaking subsequent lines. # The second line should not be ' to tree', as that's longer than the # test layout width of 8. @@ -63,7 +64,7 @@ def test_short_words_indented(self): ''', self.out.getvalue()) - def test_few_long_words_indented(self): + def test_few_long_words_indented(self) -> None: # Check wrapping in the presence of indenting. self.n._line(' '.join(['x', LONGWORD, 'y']), indent=1) self.assertEqual(' $\n'.join([' ' + 'x', @@ -71,14 +72,14 @@ def test_few_long_words_indented(self): ' ' + INDENT + 'y']) + '\n', self.out.getvalue()) - def test_escaped_spaces(self): + def test_escaped_spaces(self) -> None: self.n._line(' '.join(['x', LONGWORDWITHSPACES, 'y'])) self.assertEqual(' $\n'.join(['x', INDENT + LONGWORDWITHSPACES, INDENT + 'y']) + '\n', self.out.getvalue()) - def test_fit_many_words(self): + def test_fit_many_words(self) -> None: self.n = ninja_syntax.Writer(self.out, width=78) self.n._line('command = cd ../../chrome; python ../tools/grit/grit/format/repack.py ../out/Debug/obj/chrome/chrome_dll.gen/repack/theme_resources_large.pak ../out/Debug/gen/chrome/theme_resources_large.pak', 1) self.assertEqual('''\ @@ -88,7 +89,7 @@ def test_fit_many_words(self): ''', self.out.getvalue()) - def test_leading_space(self): + def test_leading_space(self) -> None: self.n = ninja_syntax.Writer(self.out, width=14) # force wrapping self.n.variable('foo', ['', '-bar', '-somethinglong'], 0) self.assertEqual('''\ @@ -97,7 +98,7 @@ def test_leading_space(self): ''', self.out.getvalue()) - def test_embedded_dollar_dollar(self): + def test_embedded_dollar_dollar(self) -> None: self.n = ninja_syntax.Writer(self.out, width=15) # force wrapping self.n.variable('foo', ['a$$b', '-somethinglong'], 0) self.assertEqual('''\ @@ -106,7 +107,7 @@ def test_embedded_dollar_dollar(self): ''', self.out.getvalue()) - def test_two_embedded_dollar_dollars(self): + def test_two_embedded_dollar_dollars(self) -> None: self.n = ninja_syntax.Writer(self.out, width=17) # force wrapping self.n.variable('foo', ['a$$b', '-somethinglong'], 0) self.assertEqual('''\ @@ -115,7 +116,7 @@ def test_two_embedded_dollar_dollars(self): ''', self.out.getvalue()) - def test_leading_dollar_dollar(self): + def test_leading_dollar_dollar(self) -> None: self.n = ninja_syntax.Writer(self.out, width=14) # force wrapping self.n.variable('foo', ['$$b', '-somethinglong'], 0) self.assertEqual('''\ @@ -124,7 +125,7 @@ def test_leading_dollar_dollar(self): ''', self.out.getvalue()) - def test_trailing_dollar_dollar(self): + def test_trailing_dollar_dollar(self) -> None: self.n = ninja_syntax.Writer(self.out, width=14) # force wrapping self.n.variable('foo', ['a$$', '-somethinglong'], 0) self.assertEqual('''\ @@ -134,11 +135,11 @@ def test_trailing_dollar_dollar(self): self.out.getvalue()) class TestBuild(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: self.out = StringIO() self.n = ninja_syntax.Writer(self.out) - def test_variables_dict(self): + def test_variables_dict(self) -> None: self.n.build('out', 'cc', 'in', variables={'name': 'value'}) self.assertEqual('''\ build out: cc in @@ -146,7 +147,7 @@ def test_variables_dict(self): ''', self.out.getvalue()) - def test_variables_list(self): + def test_variables_list(self) -> None: self.n.build('out', 'cc', 'in', variables=[('name', 'value')]) self.assertEqual('''\ build out: cc in @@ -154,7 +155,7 @@ def test_variables_list(self): ''', self.out.getvalue()) - def test_implicit_outputs(self): + def test_implicit_outputs(self) -> None: self.n.build('o', 'cc', 'i', implicit_outputs='io') self.assertEqual('''\ build o | io: cc i @@ -162,29 +163,29 @@ def test_implicit_outputs(self): self.out.getvalue()) class TestExpand(unittest.TestCase): - def test_basic(self): + def test_basic(self) -> None: vars = {'x': 'X'} self.assertEqual('foo', ninja_syntax.expand('foo', vars)) - def test_var(self): + def test_var(self) -> None: vars = {'xyz': 'XYZ'} self.assertEqual('fooXYZ', ninja_syntax.expand('foo$xyz', vars)) - def test_vars(self): + def test_vars(self) -> None: vars = {'x': 'X', 'y': 'YYY'} self.assertEqual('XYYY', ninja_syntax.expand('$x$y', vars)) - def test_space(self): - vars = {} + def test_space(self) -> None: + vars: Dict[str, str] = {} self.assertEqual('x y z', ninja_syntax.expand('x$ y$ z', vars)) - def test_locals(self): + def test_locals(self) -> None: vars = {'x': 'a'} local_vars = {'x': 'b'} self.assertEqual('a', ninja_syntax.expand('$x', vars)) self.assertEqual('b', ninja_syntax.expand('$x', vars, local_vars)) - def test_double(self): + def test_double(self) -> None: self.assertEqual('a b$c', ninja_syntax.expand('a$ b$$c', {})) if __name__ == '__main__': diff --git a/misc/oss-fuzz/build.sh b/misc/oss-fuzz/build.sh new file mode 100644 index 0000000000..4328feb662 --- /dev/null +++ b/misc/oss-fuzz/build.sh @@ -0,0 +1,29 @@ +#!/bin/bash -eu +# Copyright 2020 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +################################################################################ + +cmake -Bbuild-cmake -H. +cmake --build build-cmake + +cd $SRC/ninja/misc + +$CXX $CXXFLAGS -fdiagnostics-color -I/src/ninja/src -o fuzzer.o -c manifest_fuzzer.cc + +find .. -name "*.o" -exec ar rcs fuzz_lib.a {} \; + +$CXX $CXXFLAGS $LIB_FUZZING_ENGINE fuzzer.o -o $OUT/fuzzer fuzz_lib.a + +zip $OUT/fuzzer_seed_corpus.zip $SRC/sample_ninja_build diff --git a/misc/oss-fuzz/sample_ninja_build b/misc/oss-fuzz/sample_ninja_build new file mode 100644 index 0000000000..7b513be2dd --- /dev/null +++ b/misc/oss-fuzz/sample_ninja_build @@ -0,0 +1,14 @@ +# build.ninja +cc = clang +cflags = -Weverything + +rule compile + command = $cc $cflags -c $in -o $out + +rule link + command = $cc $in -o $out + +build hello.o: compile hello.c +build hello: link hello.o + +default hello diff --git a/misc/output_test.py b/misc/output_test.py index 1dcde10b03..8f3f05d8dd 100755 --- a/misc/output_test.py +++ b/misc/output_test.py @@ -11,40 +11,162 @@ import sys import tempfile import unittest +from textwrap import dedent +import typing as T default_env = dict(os.environ) -if 'NINJA_STATUS' in default_env: - del default_env['NINJA_STATUS'] -if 'CLICOLOR_FORCE' in default_env: - del default_env['CLICOLOR_FORCE'] +default_env.pop('NINJA_STATUS', None) +default_env.pop('CLICOLOR_FORCE', None) default_env['TERM'] = '' +NINJA_PATH = os.path.abspath('./ninja') -def run(build_ninja, flags='', pipe=False, env=default_env): - with tempfile.NamedTemporaryFile('w') as f: - f.write(build_ninja) - f.flush() - ninja_cmd = './ninja {} -f {}'.format(flags, f.name) +def remove_non_visible_lines(raw_output: bytes) -> str: + # When running in a smart terminal, Ninja uses CR (\r) to + # return the cursor to the start of the current line, prints + # something, then uses `\x1b[K` to clear everything until + # the end of the line. + # + # Thus printing 'FOO', 'BAR', 'ZOO' on the same line, then + # jumping to the next one results in the following output + # on Posix: + # + # '\rFOO\x1b[K\rBAR\x1b[K\rZOO\x1b[K\r\n' + # + # The following splits the output at both \r, \n and \r\n + # boundaries, which gives: + # + # [ '\r', 'FOO\x1b[K\r', 'BAR\x1b[K\r', 'ZOO\x1b[K\r\n' ] + # + decoded_lines = raw_output.decode('utf-8').splitlines(True) + + # Remove any item that ends with a '\r' as this means its + # content will be overwritten by the next item in the list. + # For the previous example, this gives: + # + # [ 'ZOO\x1b[K\r\n' ] + # + final_lines = [ l for l in decoded_lines if not l.endswith('\r') ] + + # Return a single string that concatenates all filtered lines + # while removing any remaining \r in it. Needed to transform + # \r\n into \n. + # + # "ZOO\x1b[K\n' + # + return ''.join(final_lines).replace('\r', '') + +class BuildDir: + def __init__(self, build_ninja: str): + self.build_ninja = dedent(build_ninja) + self.d = None + + def __enter__(self): + self.d = tempfile.TemporaryDirectory() + with open(os.path.join(self.d.name, 'build.ninja'), 'w') as f: + f.write(self.build_ninja) + f.flush() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.d.cleanup() + + @property + def path(self) -> str: + return os.path.realpath(self.d.name) + + + def run( + self, + flags: T.Optional[str] = None, + pipe: bool = False, + raw_output: bool = False, + env: T.Dict[str, str] = default_env, + print_err_output = True, + ) -> str: + """Run Ninja command, and get filtered output. + + Args: + flags: Extra arguments passed to Ninja. + + pipe: set to True to run Ninja in a non-interactive terminal. + If False (the default), this runs Ninja in a pty to simulate + a smart terminal (this feature cannot work on Windows!). + + raw_output: set to True to return the raw, unfiltered command + output. + + env: Optional environment dictionary to run the command in. + + print_err_output: set to False if the test expects ninja to print + something to stderr. (Otherwise, an error message from Ninja + probably represents a failed test.) + + Returns: + A UTF-8 string corresponding to the output (stdout only) of the + Ninja command. By default, partial lines that were overwritten + are removed according to the rules described in the comments + below. + """ + ninja_cmd = '{} {}'.format(NINJA_PATH, flags if flags else '') try: if pipe: - output = subprocess.check_output([ninja_cmd], shell=True, env=env) + output = subprocess.check_output( + [ninja_cmd], shell=True, cwd=self.d.name, env=env) elif platform.system() == 'Darwin': output = subprocess.check_output(['script', '-q', '/dev/null', 'bash', '-c', ninja_cmd], - env=env) + cwd=self.d.name, env=env) else: output = subprocess.check_output(['script', '-qfec', ninja_cmd, '/dev/null'], - env=env) + cwd=self.d.name, env=env) except subprocess.CalledProcessError as err: - sys.stdout.buffer.write(err.output) + if print_err_output: + sys.stdout.buffer.write(err.output) + err.cooked_output = remove_non_visible_lines(err.output) raise err - final_output = '' - for line in output.decode('utf-8').splitlines(True): - if len(line) > 0 and line[-1] == '\r': - continue - final_output += line.replace('\r', '') - return final_output + if raw_output: + return output.decode('utf-8') + return remove_non_visible_lines(output) + +def run( + build_ninja: str, + flags: T.Optional[str] = None, + pipe: bool = False, + raw_output: bool = False, + env: T.Dict[str, str] = default_env, + print_err_output = True, +) -> str: + """Run Ninja with a given build plan in a temporary directory. + """ + with BuildDir(build_ninja) as b: + return b.run(flags, pipe, raw_output, env, print_err_output) + +@unittest.skipIf(platform.system() == 'Windows', 'These test methods do not work on Windows') class Output(unittest.TestCase): - def test_issue_1418(self): + BUILD_SIMPLE_ECHO = '\n'.join(( + 'rule echo', + ' command = printf "do thing"', + ' description = echo $out', + '', + 'build a: echo', + '', + )) + + def _test_expected_error(self, plan: str, flags: T.Optional[str],expected: str, + *args, exit_code: T.Optional[int]=None, **kwargs)->None: + """Run Ninja with a given plan and flags, and verify its cooked output against an expected content. + All *args and **kwargs are passed to the `run` function + """ + actual = '' + kwargs['print_err_output'] = False + with self.assertRaises(subprocess.CalledProcessError) as cm: + run(plan, flags, *args, **kwargs) + actual = cm.exception.cooked_output + if exit_code is not None: + self.assertEqual(cm.exception.returncode, exit_code) + self.assertEqual(expected, actual) + + def test_issue_1418(self) -> None: self.assertEqual(run( '''rule echo command = sleep $delay && echo $out @@ -56,7 +178,7 @@ def test_issue_1418(self): delay = 2 build c: echo delay = 1 -'''), +''', '-j3'), '''[1/3] echo c\x1b[K c [2/3] echo b\x1b[K @@ -65,7 +187,7 @@ def test_issue_1418(self): a ''') - def test_issue_1214(self): + def test_issue_1214(self) -> None: print_red = '''rule echo command = printf '\x1b[31mred\x1b[0m' description = echo $out @@ -99,5 +221,370 @@ def test_issue_1214(self): \x1b[31mred\x1b[0m ''') + def test_issue_1966(self) -> None: + self.assertEqual(run( +'''rule cat + command = cat $rspfile $rspfile > $out + rspfile = cat.rsp + rspfile_content = a b c + +build a: cat +''', '-j3'), +'''[1/1] cat cat.rsp cat.rsp > a\x1b[K +''') + + def test_issue_2499(self) -> None: + # This verifies that Ninja prints its status line updates on a single + # line when running in a smart terminal, and when commands do not have + # any output. Get the raw command output which includes CR (\r) codes + # and all content that was printed by Ninja. + self.assertEqual(run( +'''rule touch + command = touch $out + +build foo: touch +build bar: touch foo +build zoo: touch bar +''', flags='-j1 zoo', raw_output=True).split('\r'), + [ + '', + '[0/3] touch foo\x1b[K', + '[1/3] touch foo\x1b[K', + '[1/3] touch bar\x1b[K', + '[2/3] touch bar\x1b[K', + '[2/3] touch zoo\x1b[K', + '[3/3] touch zoo\x1b[K', + '\n', + ]) + + def test_pr_1685(self) -> None: + # Running those tools without .ninja_deps and .ninja_log shouldn't fail. + self.assertEqual(run('', flags='-t recompact'), '') + self.assertEqual(run('', flags='-t restat'), '') + + def test_issue_2048(self) -> None: + with tempfile.TemporaryDirectory() as d: + with open(os.path.join(d, 'build.ninja'), 'w'): + pass + + with open(os.path.join(d, '.ninja_log'), 'w') as f: + f.write('# ninja log v4\n') + + try: + output = subprocess.check_output([NINJA_PATH, '-t', 'recompact'], + cwd=d, + env=default_env, + stderr=subprocess.STDOUT, + text=True + ) + + self.assertEqual( + output.strip(), + "ninja: warning: build log version is too old; starting over" + ) + except subprocess.CalledProcessError as err: + self.fail("non-zero exit code with: " + err.output) + + def test_pr_2540(self)->None: + py = sys.executable + plan = f'''\ +rule CUSTOM_COMMAND + command = $COMMAND + +build 124: CUSTOM_COMMAND + COMMAND = {py} -c 'exit(124)' + +build 127: CUSTOM_COMMAND + COMMAND = {py} -c 'exit(127)' + +build 130: CUSTOM_COMMAND + COMMAND = {py} -c 'exit(130)' + +build 137: CUSTOM_COMMAND + COMMAND = {py} -c 'exit(137)' + +build success: CUSTOM_COMMAND + COMMAND = sleep 0.3; echo success +''' + # Disable colors + env = default_env.copy() + env['TERM'] = 'dumb' + self._test_expected_error( + plan, '124', + f'''[1/1] {py} -c 'exit(124)' +FAILED: [code=124] 124 \n{py} -c 'exit(124)' +ninja: build stopped: subcommand failed. +''', + exit_code=124, env=env, + ) + self._test_expected_error( + plan, '127', + f'''[1/1] {py} -c 'exit(127)' +FAILED: [code=127] 127 \n{py} -c 'exit(127)' +ninja: build stopped: subcommand failed. +''', + exit_code=127, env=env, + ) + self._test_expected_error( + plan, '130', + 'ninja: build stopped: interrupted by user.\n', + exit_code=130, env=env, + ) + self._test_expected_error( + plan, '137', + f'''[1/1] {py} -c 'exit(137)' +FAILED: [code=137] 137 \n{py} -c 'exit(137)' +ninja: build stopped: subcommand failed. +''', + exit_code=137, env=env, + ) + self._test_expected_error( + plan, 'non-existent-target', + "ninja: error: unknown target 'non-existent-target'\n", + exit_code=1, env=env, + ) + self._test_expected_error( + plan, '-j2 success 127', + f'''[1/2] {py} -c 'exit(127)' +FAILED: [code=127] 127 \n{py} -c 'exit(127)' +[2/2] sleep 0.3; echo success +success +ninja: build stopped: subcommand failed. +''', + exit_code=127, env=env, + ) + + def test_depfile_directory_creation(self) -> None: + b = BuildDir('''\ + rule touch + command = touch $out && echo "$out: extra" > $depfile + + build somewhere/out: touch + depfile = somewhere_else/out.d + ''') + with b: + self.assertEqual(b.run('', pipe=True), dedent('''\ + [1/1] touch somewhere/out && echo "somewhere/out: extra" > somewhere_else/out.d + ''')) + self.assertTrue(os.path.isfile(os.path.join(b.d.name, "somewhere", "out"))) + self.assertTrue(os.path.isfile(os.path.join(b.d.name, "somewhere_else", "out.d"))) + + def test_status(self) -> None: + self.assertEqual(run(''), 'ninja: no work to do.\n') + self.assertEqual(run('', pipe=True), 'ninja: no work to do.\n') + self.assertEqual(run('', flags='--quiet'), '') + + def test_ninja_status_default(self) -> None: + 'Do we show the default status by default?' + self.assertEqual(run(Output.BUILD_SIMPLE_ECHO), '[1/1] echo a\x1b[K\ndo thing\n') + + def test_ninja_status_quiet(self) -> None: + 'Do we suppress the status information when --quiet is specified?' + output = run(Output.BUILD_SIMPLE_ECHO, flags='--quiet') + self.assertEqual(output, 'do thing\n') + + def test_entering_directory_on_stdout(self) -> None: + output = run(Output.BUILD_SIMPLE_ECHO, flags='-C$PWD', pipe=True) + self.assertEqual(output.splitlines()[0][:25], "ninja: Entering directory") + + def test_tool_inputs(self) -> None: + plan = ''' +rule cat + command = cat $in $out +build out1 : cat in1 +build out2 : cat in2 out1 +build out3 : cat out2 out1 | implicit || order_only +''' + self.assertEqual(run(plan, flags='-t inputs out3'), +'''implicit +in1 +in2 +order_only +out1 +out2 +''') + + self.assertEqual(run(plan, flags='-t inputs --dependency-order out3'), +'''in2 +in1 +out1 +out2 +implicit +order_only +''') + + # Verify that results are shell-escaped by default, unless --no-shell-escape + # is used. Also verify that phony outputs are never part of the results. + quote = '"' if platform.system() == "Windows" else "'" + + plan = ''' +rule cat + command = cat $in $out +build out1 : cat in1 +build out$ 2 : cat out1 +build out$ 3 : phony out$ 2 +build all: phony out$ 3 +''' + + # Quoting changes the order of results when sorting alphabetically. + self.assertEqual(run(plan, flags='-t inputs all'), +f'''{quote}out 2{quote} +in1 +out1 +''') + + self.assertEqual(run(plan, flags='-t inputs --no-shell-escape all'), +'''in1 +out 2 +out1 +''') + + # But not when doing dependency order. + self.assertEqual( + run( + plan, + flags='-t inputs --dependency-order all' + ), + f'''in1 +out1 +{quote}out 2{quote} +''') + + self.assertEqual( + run( + plan, + flags='-t inputs --dependency-order --no-shell-escape all' + ), + f'''in1 +out1 +out 2 +''') + + self.assertEqual( + run( + plan, + flags='-t inputs --dependency-order --no-shell-escape --print0 all' + ), + f'''in1\0out1\0out 2\0''' + ) + + + def test_tool_compdb_targets(self) -> None: + plan = ''' +rule cat + command = cat $in $out +build out1 : cat in1 +build out2 : cat in2 out1 +build out3 : cat out2 out1 +build out4 : cat in4 +''' + + + self._test_expected_error(plan, '-t compdb-targets', +'''ninja: error: compdb-targets expects the name of at least one target +usage: ninja -t compdb [-hx] target [targets] + +options: + -h display this help message + -x expand @rspfile style response file invocations +''') + + self._test_expected_error(plan, '-t compdb-targets in1', + "ninja: fatal: 'in1' is not a target (i.e. it is not an output of any `build` statement)\n") + + self._test_expected_error(plan, '-t compdb-targets nonexistent_target', + "ninja: fatal: unknown target 'nonexistent_target'\n") + + + with BuildDir(plan) as b: + actual = b.run(flags='-t compdb-targets out3') + expected = f'''[ + {{ + "directory": "{b.path}", + "command": "cat in1 out1", + "file": "in1", + "output": "out1" + }}, + {{ + "directory": "{b.path}", + "command": "cat in2 out1 out2", + "file": "in2", + "output": "out2" + }}, + {{ + "directory": "{b.path}", + "command": "cat out2 out1 out3", + "file": "out2", + "output": "out3" + }} +] +''' + self.assertEqual(expected, actual) + + + def test_tool_multi_inputs(self) -> None: + plan = ''' +rule cat + command = cat $in $out +build out1 : cat in1 +build out2 : cat in1 in2 +build out3 : cat in1 in2 in3 +''' + self.assertEqual(run(plan, flags='-t multi-inputs out1'), +'''out1in1 +'''.replace("", "\t")) + + self.assertEqual(run(plan, flags='-t multi-inputs out1 out2 out3'), +'''out1in1 +out2in1 +out2in2 +out3in1 +out3in2 +out3in3 +'''.replace("", "\t")) + + self.assertEqual(run(plan, flags='-t multi-inputs -d: out1'), +'''out1:in1 +''') + + self.assertEqual( + run( + plan, + flags='-t multi-inputs -d, --print0 out1 out2' + ), + '''out1,in1\0out2,in1\0out2,in2\0''' + ) + + + def test_explain_output(self): + b = BuildDir('''\ + build .FORCE: phony + rule create_if_non_exist + command = [ -e $out ] || touch $out + restat = true + rule write + command = cp $in $out + build input : create_if_non_exist .FORCE + build mid : write input + build output : write mid + default output + ''') + with b: + # The explain output is shown just before the relevant build: + self.assertEqual(b.run('-v -d explain'), dedent('''\ + ninja explain: .FORCE is dirty + [1/3] [ -e input ] || touch input + ninja explain: input is dirty + [2/3] cp input mid + ninja explain: mid is dirty + [3/3] cp mid output + ''')) + # Don't print "ninja explain: XXX is dirty" for inputs that are + # pruned from the graph by an earlier restat. + self.assertEqual(b.run('-v -d explain'), dedent('''\ + ninja explain: .FORCE is dirty + [1/3] [ -e input ] || touch input + ''')) + if __name__ == '__main__': unittest.main() diff --git a/misc/packaging/ninja.spec b/misc/packaging/ninja.spec index 05f5a079a3..36e5181866 100644 --- a/misc/packaging/ninja.spec +++ b/misc/packaging/ninja.spec @@ -32,7 +32,7 @@ cp -p ninja %{buildroot}%{_bindir}/ %files %defattr(-, root, root) -%doc COPYING README doc/manual.html +%doc COPYING README.md doc/manual.html %{_bindir}/* %clean diff --git a/misc/write_fake_manifests.py b/misc/write_fake_manifests.py old mode 100644 new mode 100755 index b3594de0bb..124ba8b6e0 --- a/misc/write_fake_manifests.py +++ b/misc/write_fake_manifests.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Writes large manifest files, for manifest parser performance testing. @@ -18,11 +18,12 @@ import os import random import sys +from typing import Generator, Optional, Tuple, List, Set import ninja_syntax -def paretoint(avg, alpha): +def paretoint(avg: float, alpha: float) -> int: """Returns a random integer that's avg on average, following a power law. alpha determines the shape of the power curve. alpha has to be larger than 1. The closer alpha is to 1, the higher the variation of the returned @@ -31,7 +32,7 @@ def paretoint(avg, alpha): # Based on http://neugierig.org/software/chromium/class-name-generator.html -def moar(avg_options, p_suffix): +def moar(avg_options: float, p_suffix: float) -> str: kStart = ['render', 'web', 'browser', 'tab', 'content', 'extension', 'url', 'file', 'sync', 'content', 'http', 'profile'] kOption = ['view', 'host', 'holder', 'container', 'impl', 'ref', @@ -50,46 +51,46 @@ def moar(avg_options, p_suffix): class GenRandom(object): - def __init__(self, src_dir): - self.seen_names = set([None]) - self.seen_defines = set([None]) + def __init__(self, src_dir: str) -> None: + self.seen_names: Set[Optional[str]] = set([None]) + self.seen_defines: Set[Optional[str]] = set([None]) self.src_dir = src_dir - def _unique_string(self, seen, avg_options=1.3, p_suffix=0.1): + def _unique_string(self, seen: Set[Optional[str]], avg_options: float = 1.3, p_suffix: float = 0.1) -> str: s = None while s in seen: s = moar(avg_options, p_suffix) seen.add(s) - return s + return s # type: ignore # Incompatible return value type - def _n_unique_strings(self, n): - seen = set([None]) + def _n_unique_strings(self, n: int) -> List[str]: + seen: Set[Optional[str]] = set([None]) return [self._unique_string(seen, avg_options=3, p_suffix=0.4) - for _ in xrange(n)] + for _ in range(n)] - def target_name(self): + def target_name(self) -> str: return self._unique_string(p_suffix=0, seen=self.seen_names) - def path(self): + def path(self) -> str: return os.path.sep.join([ self._unique_string(self.seen_names, avg_options=1, p_suffix=0) - for _ in xrange(1 + paretoint(0.6, alpha=4))]) + for _ in range(1 + paretoint(0.6, alpha=4))]) - def src_obj_pairs(self, path, name): + def src_obj_pairs(self, path: str, name: str) -> List[Tuple[str, str]]: num_sources = paretoint(55, alpha=2) + 1 return [(os.path.join(self.src_dir, path, s + '.cc'), os.path.join('obj', path, '%s.%s.o' % (name, s))) for s in self._n_unique_strings(num_sources)] - def defines(self): + def defines(self) -> List[str]: return [ '-DENABLE_' + self._unique_string(self.seen_defines).upper() - for _ in xrange(paretoint(20, alpha=3))] + for _ in range(paretoint(20, alpha=3))] LIB, EXE = 0, 1 class Target(object): - def __init__(self, gen, kind): + def __init__(self, gen: GenRandom, kind: int) -> None: self.name = gen.target_name() self.dir_path = gen.path() self.ninja_file_path = os.path.join( @@ -100,12 +101,12 @@ def __init__(self, gen, kind): elif kind == EXE: self.output = os.path.join(self.name) self.defines = gen.defines() - self.deps = [] + self.deps: List[Target] = [] self.kind = kind self.has_compile_depends = random.random() < 0.4 -def write_target_ninja(ninja, target, src_dir): +def write_target_ninja(ninja: ninja_syntax.Writer, target: Target, src_dir: str) -> None: compile_depends = None if target.has_compile_depends: compile_depends = os.path.join( @@ -133,7 +134,7 @@ def write_target_ninja(ninja, target, src_dir): implicit=deps) -def write_sources(target, root_dir): +def write_sources(target: Target, root_dir: str) -> None: need_main = target.kind == EXE includes = [] @@ -174,7 +175,7 @@ def write_sources(target, root_dir): f.write('int main(int argc, char **argv) {}\n') need_main = False -def write_master_ninja(master_ninja, targets): +def write_master_ninja(master_ninja: ninja_syntax.Writer, targets: List[Target]) -> None: """Writes master build.ninja file, referencing all given subninjas.""" master_ninja.variable('cxx', 'c++') master_ninja.variable('ld', '$cxx') @@ -212,7 +213,7 @@ def write_master_ninja(master_ninja, targets): @contextlib.contextmanager -def FileWriter(path): +def FileWriter(path: str) -> Generator[ninja_syntax.Writer, None, None]: """Context manager for a ninja_syntax object writing to a file.""" try: os.makedirs(os.path.dirname(path)) @@ -223,11 +224,11 @@ def FileWriter(path): f.close() -def random_targets(num_targets, src_dir): +def random_targets(num_targets: int, src_dir: str) -> List[Target]: gen = GenRandom(src_dir) # N-1 static libraries, and 1 executable depending on all of them. - targets = [Target(gen, LIB) for i in xrange(num_targets - 1)] + targets = [Target(gen, LIB) for i in range(num_targets - 1)] for i in range(len(targets)): targets[i].deps = [t for t in targets[0:i] if random.random() < 0.05] @@ -238,7 +239,7 @@ def random_targets(num_targets, src_dir): return targets -def main(): +def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('-s', '--sources', nargs="?", const="src", help='write sources to directory (relative to output directory)') @@ -269,4 +270,4 @@ def main(): if __name__ == '__main__': - sys.exit(main()) + sys.exit(main()) # type: ignore # "main" does not return a value diff --git a/misc/zsh-completion b/misc/zsh-completion index 4cee3b8631..d439df3994 100644 --- a/misc/zsh-completion +++ b/misc/zsh-completion @@ -16,7 +16,7 @@ # Add the following to your .zshrc to tab-complete ninja targets # fpath=(path/to/ninja/misc/zsh-completion $fpath) -__get_targets() { +(( $+functions[_ninja-get-targets] )) || _ninja-get-targets() { dir="." if [ -n "${opt_args[-C]}" ]; then @@ -31,42 +31,45 @@ __get_targets() { eval ${targets_command} 2>/dev/null | cut -d: -f1 } -__get_tools() { - ninja -t list 2>/dev/null | while read -r a b; do echo $a; done | tail -n +2 +(( $+functions[_ninja-get-tools] )) || _ninja-get-tools() { + # remove the first line; remove the leading spaces; replace spaces with colon + ninja -t list 2> /dev/null | sed -e '1d;s/^ *//;s/ \+/:/' } -__get_modes() { - ninja -d list 2>/dev/null | while read -r a b; do echo $a; done | tail -n +2 | sed '$d' +(( $+functions[_ninja-get-modes] )) || _ninja-get-modes() { + # remove the first line; remove the last line; remove the leading spaces; replace spaces with colon + ninja -d list 2> /dev/null | sed -e '1d;$d;s/^ *//;s/ \+/:/' } -__modes() { +(( $+functions[_ninja-modes] )) || _ninja-modes() { local -a modes - modes=(${(fo)"$(__get_modes)"}) + modes=(${(fo)"$(_ninja-get-modes)"}) _describe 'modes' modes } -__tools() { +(( $+functions[_ninja-tools] )) || _ninja-tools() { local -a tools - tools=(${(fo)"$(__get_tools)"}) + tools=(${(fo)"$(_ninja-get-tools)"}) _describe 'tools' tools } -__targets() { +(( $+functions[_ninja-targets] )) || _ninja-targets() { local -a targets - targets=(${(fo)"$(__get_targets)"}) + targets=(${(fo)"$(_ninja-get-targets)"}) _describe 'targets' targets } _arguments \ - {-h,--help}'[Show help]' \ - '--version[Print ninja version]' \ + '(- *)'{-h,--help}'[Show help]' \ + '(- *)--version[Print ninja version]' \ '-C+[Change to directory before doing anything else]:directories:_directories' \ '-f+[Specify input build file (default=build.ninja)]:files:_files' \ '-j+[Run N jobs in parallel (default=number of CPUs available)]:number of jobs' \ '-l+[Do not start new jobs if the load average is greater than N]:number of jobs' \ '-k+[Keep going until N jobs fail (default=1)]:number of jobs' \ '-n[Dry run (do not run commands but act like they succeeded)]' \ - '-v[Show all command lines while building]' \ - '-d+[Enable debugging (use -d list to list modes)]:modes:__modes' \ - '-t+[Run a subtool (use -t list to list subtools)]:tools:__tools' \ - '*::targets:__targets' + '(-v --verbose --quiet)'{-v,--verbose}'[Show all command lines while building]' \ + "(-v --verbose --quiet)--quiet[Don't show progress status, just command output]" \ + '-d+[Enable debugging (use -d list to list modes)]:modes:_ninja-modes' \ + '-t+[Run a subtool (use -t list to list subtools)]:tools:_ninja-tools' \ + '*::targets:_ninja-targets' diff --git a/src/browse.cc b/src/browse.cc index c08c9f4d40..2956f896c4 100644 --- a/src/browse.cc +++ b/src/browse.cc @@ -22,6 +22,8 @@ #include "build/browse_py.h" +using namespace std; + void RunBrowsePython(State* state, const char* ninja_command, const char* input_file, int argc, char* argv[]) { // Fork off a Python process and have it run our code via its stdin. @@ -57,7 +59,7 @@ void RunBrowsePython(State* state, const char* ninja_command, command.push_back(argv[i]); } command.push_back(NULL); - execvp(command[0], (char**)&command[0]); + execvp(command[0], const_cast(&command[0])); if (errno == ENOENT) { printf("ninja: %s is required for the browse tool\n", NINJA_PYTHON); } else { @@ -69,8 +71,13 @@ void RunBrowsePython(State* state, const char* ninja_command, close(pipefd[0]); // Write the script file into the stdin of the Python process. - ssize_t len = write(pipefd[1], kBrowsePy, sizeof(kBrowsePy)); - if (len < (ssize_t)sizeof(kBrowsePy)) + // Only write n - 1 bytes, because Python 3.11 does not allow null + // bytes in source code anymore, so avoid writing the null string + // terminator. + // See https://github.com/python/cpython/issues/96670 + auto kBrowsePyLength = sizeof(kBrowsePy) - 1; + ssize_t len = write(pipefd[1], kBrowsePy, kBrowsePyLength); + if (len < (ssize_t)kBrowsePyLength) perror("ninja: write"); close(pipefd[1]); exit(0); diff --git a/src/browse.py b/src/browse.py index 1c9c39b8e8..9dead142ba 100755 --- a/src/browse.py +++ b/src/browse.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright 2001 Google Inc. All Rights Reserved. # @@ -20,26 +20,28 @@ it when needed. """ -from __future__ import print_function - try: import http.server as httpserver import socketserver except ImportError: - import BaseHTTPServer as httpserver - import SocketServer as socketserver + import BaseHTTPServer as httpserver # type: ignore # Name "httpserver" already defined + import SocketServer as socketserver # type: ignore # Name "socketserver" already defined import argparse -import cgi import os import socket import subprocess import sys import webbrowser +if sys.version_info >= (3, 2): + from html import escape +else: + from cgi import escape try: - from urllib.request import unquote + from urllib.request import unquote # type: ignore # Module "urllib.request" has no attribute "unquote" except ImportError: from urllib2 import unquote from collections import namedtuple +from typing import Tuple, Any Node = namedtuple('Node', ['inputs', 'rule', 'target', 'outputs']) @@ -56,15 +58,15 @@ # This means there's no single view that shows you all inputs and outputs # of an edge. But I think it's less confusing than alternatives. -def match_strip(line, prefix): +def match_strip(line: str, prefix: str) -> Tuple[bool, str]: if not line.startswith(prefix): return (False, line) return (True, line[len(prefix):]) -def html_escape(text): - return cgi.escape(text, quote=True) +def html_escape(text: str) -> str: + return escape(text, quote=True) -def parse(text): +def parse(text: str) -> Node: lines = iter(text.split('\n')) target = None @@ -101,7 +103,7 @@ def parse(text): return Node(inputs, rule, target, outputs) -def create_page(body): +def create_page(body: str) -> str: return ''' ''' + body -def generate_html(node): +def generate_html(node: Node) -> str: document = ['

%s

' % html_escape(node.target)] if node.inputs: @@ -155,14 +157,14 @@ def generate_html(node): return '\n'.join(document) -def ninja_dump(target): +def ninja_dump(target: str) -> Tuple[str, str, int]: cmd = [args.ninja_command, '-f', args.f, '-t', 'query', target] proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) return proc.communicate() + (proc.returncode,) class RequestHandler(httpserver.BaseHTTPRequestHandler): - def do_GET(self): + def do_GET(self) -> None: assert self.path[0] == '/' target = unquote(self.path[1:]) @@ -189,7 +191,7 @@ def do_GET(self): self.end_headers() self.wfile.write(create_page(page_body).encode('utf-8')) - def log_message(self, format, *args): + def log_message(self, format: str, *args: Any) -> None: pass # Swallow console spam. parser = argparse.ArgumentParser(prog='ninja -t browse') diff --git a/src/build.cc b/src/build.cc index a0557382e0..9411824f3a 100644 --- a/src/build.cc +++ b/src/build.cc @@ -16,14 +16,13 @@ #include #include +#include #include #include -#include -#ifdef _WIN32 -#include -#include -#endif +#include +#include +#include #if defined(__SVR4) && defined(__sun) #include @@ -35,28 +34,31 @@ #include "depfile_parser.h" #include "deps_log.h" #include "disk_interface.h" +#include "exit_status.h" +#include "explanations.h" #include "graph.h" +#include "metrics.h" #include "state.h" -#include "subprocess.h" +#include "status.h" #include "util.h" +using namespace std; + namespace { /// A CommandRunner that doesn't actually run the commands. struct DryRunCommandRunner : public CommandRunner { - virtual ~DryRunCommandRunner() {} - // Overridden from CommandRunner: - virtual bool CanRunMore(); - virtual bool StartCommand(Edge* edge); - virtual bool WaitForCommand(Result* result); + size_t CanRunMore() const override; + bool StartCommand(Edge* edge) override; + bool WaitForCommand(Result* result) override; private: queue finished_; }; -bool DryRunCommandRunner::CanRunMore() { - return true; +size_t DryRunCommandRunner::CanRunMore() const { + return SIZE_MAX; } bool DryRunCommandRunner::StartCommand(Edge* edge) { @@ -76,232 +78,6 @@ bool DryRunCommandRunner::WaitForCommand(Result* result) { } // namespace -BuildStatus::BuildStatus(const BuildConfig& config) - : config_(config), - start_time_millis_(GetTimeMillis()), - started_edges_(0), finished_edges_(0), total_edges_(0), - progress_status_format_(NULL), - overall_rate_(), current_rate_(config.parallelism) { - - // Don't do anything fancy in verbose mode. - if (config_.verbosity != BuildConfig::NORMAL) - printer_.set_smart_terminal(false); - - progress_status_format_ = getenv("NINJA_STATUS"); - if (!progress_status_format_) - progress_status_format_ = "[%f/%t] "; -} - -void BuildStatus::PlanHasTotalEdges(int total) { - total_edges_ = total; -} - -void BuildStatus::BuildEdgeStarted(Edge* edge) { - assert(running_edges_.find(edge) == running_edges_.end()); - int start_time = (int)(GetTimeMillis() - start_time_millis_); - running_edges_.insert(make_pair(edge, start_time)); - ++started_edges_; - - if (edge->use_console() || printer_.is_smart_terminal()) - PrintStatus(edge, kEdgeStarted); - - if (edge->use_console()) - printer_.SetConsoleLocked(true); -} - -void BuildStatus::BuildEdgeFinished(Edge* edge, - bool success, - const string& output, - int* start_time, - int* end_time) { - int64_t now = GetTimeMillis(); - - ++finished_edges_; - - RunningEdgeMap::iterator i = running_edges_.find(edge); - *start_time = i->second; - *end_time = (int)(now - start_time_millis_); - running_edges_.erase(i); - - if (edge->use_console()) - printer_.SetConsoleLocked(false); - - if (config_.verbosity == BuildConfig::QUIET) - return; - - if (!edge->use_console()) - PrintStatus(edge, kEdgeFinished); - - // Print the command that is spewing before printing its output. - if (!success) { - string outputs; - for (vector::const_iterator o = edge->outputs_.begin(); - o != edge->outputs_.end(); ++o) - outputs += (*o)->path() + " "; - - printer_.PrintOnNewLine("FAILED: " + outputs + "\n"); - printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n"); - } - - if (!output.empty()) { - // ninja sets stdout and stderr of subprocesses to a pipe, to be able to - // check if the output is empty. Some compilers, e.g. clang, check - // isatty(stderr) to decide if they should print colored output. - // To make it possible to use colored output with ninja, subprocesses should - // be run with a flag that forces them to always print color escape codes. - // To make sure these escape codes don't show up in a file if ninja's output - // is piped to a file, ninja strips ansi escape codes again if it's not - // writing to a |smart_terminal_|. - // (Launching subprocesses in pseudo ttys doesn't work because there are - // only a few hundred available on some systems, and ninja can launch - // thousands of parallel compile commands.) - string final_output; - if (!printer_.supports_color()) - final_output = StripAnsiEscapeCodes(output); - else - final_output = output; - -#ifdef _WIN32 - // Fix extra CR being added on Windows, writing out CR CR LF (#773) - _setmode(_fileno(stdout), _O_BINARY); // Begin Windows extra CR fix -#endif - - printer_.PrintOnNewLine(final_output); - -#ifdef _WIN32 - _setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix -#endif - } -} - -void BuildStatus::BuildLoadDyndeps() { - // The DependencyScan calls EXPLAIN() to print lines explaining why - // it considers a portion of the graph to be out of date. Normally - // this is done before the build starts, but our caller is about to - // load a dyndep file during the build. Doing so may generate more - // exlanation lines (via fprintf directly to stderr), but in an - // interactive console the cursor is currently at the end of a status - // line. Start a new line so that the first explanation does not - // append to the status line. After the explanations are done a - // new build status line will appear. - if (g_explaining) - printer_.PrintOnNewLine(""); -} - -void BuildStatus::BuildStarted() { - overall_rate_.Restart(); - current_rate_.Restart(); -} - -void BuildStatus::BuildFinished() { - printer_.SetConsoleLocked(false); - printer_.PrintOnNewLine(""); -} - -string BuildStatus::FormatProgressStatus( - const char* progress_status_format, EdgeStatus status) const { - string out; - char buf[32]; - int percent; - for (const char* s = progress_status_format; *s != '\0'; ++s) { - if (*s == '%') { - ++s; - switch (*s) { - case '%': - out.push_back('%'); - break; - - // Started edges. - case 's': - snprintf(buf, sizeof(buf), "%d", started_edges_); - out += buf; - break; - - // Total edges. - case 't': - snprintf(buf, sizeof(buf), "%d", total_edges_); - out += buf; - break; - - // Running edges. - case 'r': { - int running_edges = started_edges_ - finished_edges_; - // count the edge that just finished as a running edge - if (status == kEdgeFinished) - running_edges++; - snprintf(buf, sizeof(buf), "%d", running_edges); - out += buf; - break; - } - - // Unstarted edges. - case 'u': - snprintf(buf, sizeof(buf), "%d", total_edges_ - started_edges_); - out += buf; - break; - - // Finished edges. - case 'f': - snprintf(buf, sizeof(buf), "%d", finished_edges_); - out += buf; - break; - - // Overall finished edges per second. - case 'o': - overall_rate_.UpdateRate(finished_edges_); - SnprintfRate(overall_rate_.rate(), buf, "%.1f"); - out += buf; - break; - - // Current rate, average over the last '-j' jobs. - case 'c': - current_rate_.UpdateRate(finished_edges_); - SnprintfRate(current_rate_.rate(), buf, "%.1f"); - out += buf; - break; - - // Percentage - case 'p': - percent = (100 * finished_edges_) / total_edges_; - snprintf(buf, sizeof(buf), "%3i%%", percent); - out += buf; - break; - - case 'e': { - double elapsed = overall_rate_.Elapsed(); - snprintf(buf, sizeof(buf), "%.3f", elapsed); - out += buf; - break; - } - - default: - Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s); - return ""; - } - } else { - out.push_back(*s); - } - } - - return out; -} - -void BuildStatus::PrintStatus(Edge* edge, EdgeStatus status) { - if (config_.verbosity == BuildConfig::QUIET) - return; - - bool force_full_command = config_.verbosity == BuildConfig::VERBOSE; - - string to_print = edge->GetBinding("description"); - if (to_print.empty() || force_full_command) - to_print = edge->GetBinding("command"); - - to_print = FormatProgressStatus(progress_status_format_, status) + to_print; - - printer_.Print(to_print, - force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE); -} - Plan::Plan(Builder* builder) : builder_(builder) , command_edges_(0) @@ -315,22 +91,28 @@ void Plan::Reset() { want_.clear(); } -bool Plan::AddTarget(Node* node, string* err) { - return AddSubTarget(node, NULL, err, NULL); +bool Plan::AddTarget(const Node* target, string* err) { + targets_.push_back(target); + return AddSubTarget(target, NULL, err, NULL); } -bool Plan::AddSubTarget(Node* node, Node* dependent, string* err, +bool Plan::AddSubTarget(const Node* node, const Node* dependent, string* err, set* dyndep_walk) { Edge* edge = node->in_edge(); - if (!edge) { // Leaf node. - if (node->dirty()) { - string referenced; - if (dependent) - referenced = ", needed by '" + dependent->path() + "',"; - *err = "'" + node->path() + "'" + referenced + " missing " - "and no known rule to make it"; - } - return false; + if (!edge) { + // Leaf node, this can be either a regular input from the manifest + // (e.g. a source file), or an implicit input from a depfile or dyndep + // file. In the first case, a dirty flag means the file is missing, + // and the build should stop. In the second, do not do anything here + // since there is no producing edge to add to the plan. + if (node->dirty() && !node->generated_by_dep_loader()) { + string referenced; + if (dependent) + referenced = ", needed by '" + dependent->path() + "',"; + *err = "'" + node->path() + "'" + referenced + + " missing and no known rule to make it"; + } + return false; } if (edge->outputs_ready()) @@ -350,8 +132,6 @@ bool Plan::AddSubTarget(Node* node, Node* dependent, string* err, if (node->dirty() && want == kWantNothing) { want = kWantToStart; EdgeWanted(edge); - if (!dyndep_walk && edge->AllInputsReady()) - ScheduleWork(want_ins.first); } if (dyndep_walk) @@ -369,19 +149,22 @@ bool Plan::AddSubTarget(Node* node, Node* dependent, string* err, return true; } -void Plan::EdgeWanted(Edge* edge) { +void Plan::EdgeWanted(const Edge* edge) { ++wanted_edges_; - if (!edge->is_phony()) + if (!edge->is_phony()) { ++command_edges_; + if (builder_) + builder_->status_->EdgeAddedToPlan(edge); + } } Edge* Plan::FindWork() { if (ready_.empty()) return NULL; - set::iterator e = ready_.begin(); - Edge* edge = *e; - ready_.erase(e); - return edge; + + Edge* work = ready_.top(); + ready_.pop(); + return work; } void Plan::ScheduleWork(map::iterator want_e) { @@ -402,7 +185,7 @@ void Plan::ScheduleWork(map::iterator want_e) { pool->RetrieveReadyEdges(&ready_); } else { pool->EdgeScheduled(*edge); - ready_.insert(edge); + ready_.push(edge); } } @@ -521,15 +304,18 @@ bool Plan::CleanNode(DependencyScan* scan, Node* node, string* err) { want_e->second = kWantNothing; --wanted_edges_; - if (!(*oe)->is_phony()) + if (!(*oe)->is_phony()) { --command_edges_; + if (builder_) + builder_->status_->EdgeRemovedFromPlan(*oe); + } } } } return true; } -bool Plan::DyndepsLoaded(DependencyScan* scan, Node* node, +bool Plan::DyndepsLoaded(DependencyScan* scan, const Node* node, const DyndepFile& ddf, string* err) { // Recompute the dirty state of all our direct and indirect dependents now // that our dyndep information has been loaded. @@ -597,7 +383,7 @@ bool Plan::DyndepsLoaded(DependencyScan* scan, Node* node, return true; } -bool Plan::RefreshDyndepDependents(DependencyScan* scan, Node* node, +bool Plan::RefreshDyndepDependents(DependencyScan* scan, const Node* node, string* err) { // Collect the transitive closure of dependents and mark their edges // as not yet visited by RecomputeDirty. @@ -611,8 +397,21 @@ bool Plan::RefreshDyndepDependents(DependencyScan* scan, Node* node, Node* n = *i; // Check if this dependent node is now dirty. Also checks for new cycles. - if (!scan->RecomputeDirty(n, err)) + std::vector validation_nodes; + if (!scan->RecomputeDirty(n, &validation_nodes, err)) return false; + + // Add any validation nodes found during RecomputeDirty as new top level + // targets. + for (std::vector::iterator v = validation_nodes.begin(); + v != validation_nodes.end(); ++v) { + if (Edge* in_edge = (*v)->in_edge()) { + if (!in_edge->outputs_ready() && + !AddTarget(*v, err)) { + return false; + } + } + } if (!n->dirty()) continue; @@ -631,7 +430,7 @@ bool Plan::RefreshDyndepDependents(DependencyScan* scan, Node* node, return true; } -void Plan::UnmarkDependents(Node* node, set* dependents) { +void Plan::UnmarkDependents(const Node* node, set* dependents) { for (vector::const_iterator oe = node->out_edges().begin(); oe != node->out_edges().end(); ++oe) { Edge* edge = *oe; @@ -651,91 +450,165 @@ void Plan::UnmarkDependents(Node* node, set* dependents) { } } -void Plan::Dump() { - printf("pending: %d\n", (int)want_.size()); - for (map::iterator e = want_.begin(); e != want_.end(); ++e) { - if (e->second != kWantNothing) - printf("want "); - e->first->Dump(); - } - printf("ready: %d\n", (int)ready_.size()); +namespace { + +// Heuristic for edge priority weighting. +// Phony edges are free (0 cost), all other edges are weighted equally. +int64_t EdgeWeightHeuristic(Edge *edge) { + return edge->is_phony() ? 0 : 1; } -struct RealCommandRunner : public CommandRunner { - explicit RealCommandRunner(const BuildConfig& config) : config_(config) {} - virtual ~RealCommandRunner() {} - virtual bool CanRunMore(); - virtual bool StartCommand(Edge* edge); - virtual bool WaitForCommand(Result* result); - virtual vector GetActiveEdges(); - virtual void Abort(); - - const BuildConfig& config_; - SubprocessSet subprocs_; - map subproc_to_edge_; -}; +} // namespace -vector RealCommandRunner::GetActiveEdges() { - vector edges; - for (map::iterator e = subproc_to_edge_.begin(); - e != subproc_to_edge_.end(); ++e) - edges.push_back(e->second); - return edges; -} +void Plan::ComputeCriticalPath() { + METRIC_RECORD("ComputeCriticalPath"); + + // Convenience class to perform a topological sort of all edges + // reachable from a set of unique targets. Usage is: + // + // 1) Create instance. + // + // 2) Call VisitTarget() as many times as necessary. + // Note that duplicate targets are properly ignored. + // + // 3) Call result() to get a sorted list of edges, + // where each edge appears _after_ its parents, + // i.e. the edges producing its inputs, in the list. + // + struct TopoSort { + void VisitTarget(const Node* target) { + Edge* producer = target->in_edge(); + if (producer) + Visit(producer); + } -void RealCommandRunner::Abort() { - subprocs_.Clear(); -} + const std::vector& result() const { return sorted_edges_; } + + private: + // Implementation note: + // + // This is the regular depth-first-search algorithm described + // at https://en.wikipedia.org/wiki/Topological_sorting, except + // that: + // + // - Edges are appended to the end of the list, for performance + // reasons. Hence the order used in result(). + // + // - Since the graph cannot have any cycles, temporary marks + // are not necessary, and a simple set is used to record + // which edges have already been visited. + // + void Visit(Edge* edge) { + auto insertion = visited_set_.emplace(edge); + if (!insertion.second) + return; + + for (const Node* input : edge->inputs_) { + Edge* producer = input->in_edge(); + if (producer) + Visit(producer); + } + sorted_edges_.push_back(edge); + } -bool RealCommandRunner::CanRunMore() { - size_t subproc_number = - subprocs_.running_.size() + subprocs_.finished_.size(); - return (int)subproc_number < config_.parallelism - && ((subprocs_.running_.empty() || config_.max_load_average <= 0.0f) - || GetLoadAverage() < config_.max_load_average); -} + std::unordered_set visited_set_; + std::vector sorted_edges_; + }; -bool RealCommandRunner::StartCommand(Edge* edge) { - string command = edge->EvaluateCommand(); - Subprocess* subproc = subprocs_.Add(command, edge->use_console()); - if (!subproc) - return false; - subproc_to_edge_.insert(make_pair(subproc, edge)); + TopoSort topo_sort; + for (const Node* target : targets_) { + topo_sort.VisitTarget(target); + } - return true; + const auto& sorted_edges = topo_sort.result(); + + // First, reset all weights to 1. + for (Edge* edge : sorted_edges) + edge->set_critical_path_weight(EdgeWeightHeuristic(edge)); + + // Second propagate / increment weights from + // children to parents. Scan the list + // in reverse order to do so. + for (auto reverse_it = sorted_edges.rbegin(); + reverse_it != sorted_edges.rend(); ++reverse_it) { + Edge* edge = *reverse_it; + int64_t edge_weight = edge->critical_path_weight(); + + for (const Node* input : edge->inputs_) { + Edge* producer = input->in_edge(); + if (!producer) + continue; + + int64_t producer_weight = producer->critical_path_weight(); + int64_t candidate_weight = edge_weight + EdgeWeightHeuristic(producer); + if (candidate_weight > producer_weight) + producer->set_critical_path_weight(candidate_weight); + } + } } -bool RealCommandRunner::WaitForCommand(Result* result) { - Subprocess* subproc; - while ((subproc = subprocs_.NextFinished()) == NULL) { - bool interrupted = subprocs_.DoWork(); - if (interrupted) - return false; +void Plan::ScheduleInitialEdges() { + // Add ready edges to queue. + assert(ready_.empty()); + std::set pools; + + for (std::map::iterator it = want_.begin(), + end = want_.end(); it != end; ++it) { + Edge* edge = it->first; + Plan::Want want = it->second; + if (want == kWantToStart && edge->AllInputsReady()) { + Pool* pool = edge->pool(); + if (pool->ShouldDelayEdge()) { + pool->DelayEdge(edge); + pools.insert(pool); + } else { + ScheduleWork(it); + } + } } - result->status = subproc->Finish(); - result->output = subproc->GetOutput(); + // Call RetrieveReadyEdges only once at the end so higher priority + // edges are retrieved first, not the ones that happen to be first + // in the want_ map. + for (std::set::iterator it=pools.begin(), + end = pools.end(); it != end; ++it) { + (*it)->RetrieveReadyEdges(&ready_); + } +} - map::iterator e = subproc_to_edge_.find(subproc); - result->edge = e->second; - subproc_to_edge_.erase(e); +void Plan::PrepareQueue() { + ComputeCriticalPath(); + ScheduleInitialEdges(); +} - delete subproc; - return true; +void Plan::Dump() const { + printf("pending: %d\n", (int)want_.size()); + for (map::const_iterator e = want_.begin(); e != want_.end(); ++e) { + if (e->second != kWantNothing) + printf("want "); + e->first->Dump(); + } + printf("ready: %d\n", (int)ready_.size()); } -Builder::Builder(State* state, const BuildConfig& config, - BuildLog* build_log, DepsLog* deps_log, - DiskInterface* disk_interface) - : state_(state), config_(config), - plan_(this), disk_interface_(disk_interface), +Builder::Builder(State* state, const BuildConfig& config, BuildLog* build_log, + DepsLog* deps_log, DiskInterface* disk_interface, + Status* status, int64_t start_time_millis) + : state_(state), config_(config), plan_(this), status_(status), + start_time_millis_(start_time_millis), disk_interface_(disk_interface), + explanations_(g_explaining ? new Explanations() : nullptr), scan_(state, build_log, deps_log, disk_interface, - &config_.depfile_parser_options) { - status_ = new BuildStatus(config); + &config_.depfile_parser_options, explanations_.get()) { + lock_file_path_ = ".ninja_lock"; + string build_dir = state_->bindings_.LookupVariable("builddir"); + if (!build_dir.empty()) + lock_file_path_ = build_dir + "/" + lock_file_path_; + status_->SetExplanations(explanations_.get()); } Builder::~Builder() { Cleanup(); + status_->SetExplanations(nullptr); } void Builder::Cleanup() { @@ -758,7 +631,7 @@ void Builder::Cleanup() { string err; TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), &err); if (new_mtime == -1) // Log and ignore Stat() errors. - Error("%s", err.c_str()); + status_->Error("%s", err.c_str()); if (!depfile.empty() || (*o)->mtime() != new_mtime) disk_interface_->RemoveFile((*o)->path()); } @@ -766,6 +639,10 @@ void Builder::Cleanup() { disk_interface_->RemoveFile(depfile); } } + + string err; + if (disk_interface_->Stat(lock_file_path_, &err) > 0) + disk_interface_->RemoveFile(lock_file_path_); } Node* Builder::AddTarget(const string& name, string* err) { @@ -779,17 +656,29 @@ Node* Builder::AddTarget(const string& name, string* err) { return node; } -bool Builder::AddTarget(Node* node, string* err) { - if (!scan_.RecomputeDirty(node, err)) +bool Builder::AddTarget(Node* target, string* err) { + std::vector validation_nodes; + if (!scan_.RecomputeDirty(target, &validation_nodes, err)) return false; - if (Edge* in_edge = node->in_edge()) { - if (in_edge->outputs_ready()) - return true; // Nothing to do. + Edge* in_edge = target->in_edge(); + if (!in_edge || !in_edge->outputs_ready()) { + if (!plan_.AddTarget(target, err)) { + return false; + } } - if (!plan_.AddTarget(node, err)) - return false; + // Also add any validation nodes found during RecomputeDirty as top level + // targets. + for (std::vector::iterator n = validation_nodes.begin(); + n != validation_nodes.end(); ++n) { + if (Edge* validation_in_edge = (*n)->in_edge()) { + if (!validation_in_edge->outputs_ready() && + !plan_.AddTarget(*n, err)) { + return false; + } + } + } return true; } @@ -798,10 +687,10 @@ bool Builder::AlreadyUpToDate() const { return !plan_.more_to_do(); } -bool Builder::Build(string* err) { +ExitStatus Builder::Build(string* err) { assert(!AlreadyUpToDate()); + plan_.PrepareQueue(); - status_->PlanHasTotalEdges(plan_.command_edge_count()); int pending_commands = 0; int failures_allowed = config_.failures_allowed; @@ -810,7 +699,7 @@ bool Builder::Build(string* err) { if (config_.dry_run) command_runner_.reset(new DryRunCommandRunner); else - command_runner_.reset(new RealCommandRunner(config_)); + command_runner_.reset(CommandRunner::factory(config_)); } // We are about to start the build process. @@ -823,27 +712,44 @@ bool Builder::Build(string* err) { // Second, we attempt to wait for / reap the next finished command. while (plan_.more_to_do()) { // See if we can start any more commands. - if (failures_allowed && command_runner_->CanRunMore()) { - if (Edge* edge = plan_.FindWork()) { + if (failures_allowed) { + size_t capacity = command_runner_->CanRunMore(); + while (capacity > 0) { + Edge* edge = plan_.FindWork(); + if (!edge) + break; + + if (edge->GetBindingBool("generator")) { + scan_.build_log()->Close(); + } + if (!StartEdge(edge, err)) { Cleanup(); status_->BuildFinished(); - return false; + return ExitFailure; } if (edge->is_phony()) { if (!plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, err)) { Cleanup(); status_->BuildFinished(); - return false; + return ExitFailure; } } else { ++pending_commands; - } - // We made some progress; go back to the main loop. - continue; + --capacity; + + // Re-evaluate capacity. + size_t current_capacity = command_runner_->CanRunMore(); + if (current_capacity < capacity) + capacity = current_capacity; + } } + + // We are finished with all work items and have no pending + // commands. Therefore, break out of the main loop. + if (pending_commands == 0 && !plan_.more_to_do()) break; } // See if we can reap any finished commands. @@ -854,14 +760,16 @@ bool Builder::Build(string* err) { Cleanup(); status_->BuildFinished(); *err = "interrupted by user"; - return false; + return result.status; } --pending_commands; - if (!FinishCommand(&result, err)) { + bool command_finished = FinishCommand(&result, err); + SetFailureCode(result.status); + if (!command_finished) { Cleanup(); status_->BuildFinished(); - return false; + return result.status; } if (!result.success()) { @@ -885,11 +793,11 @@ bool Builder::Build(string* err) { else *err = "stuck [this is a bug]"; - return false; + return GetExitCode(); } status_->BuildFinished(); - return true; + return ExitSuccess; } bool Builder::StartEdge(Edge* edge, string* err) { @@ -897,16 +805,36 @@ bool Builder::StartEdge(Edge* edge, string* err) { if (edge->is_phony()) return true; - status_->BuildEdgeStarted(edge); + int64_t start_time_millis = GetTimeMillis() - start_time_millis_; + running_edges_.insert(make_pair(edge, start_time_millis)); - // Create directories necessary for outputs. + status_->BuildEdgeStarted(edge, start_time_millis); + + TimeStamp build_start = config_.dry_run ? 0 : -1; + + // Create directories necessary for outputs and remember the current + // filesystem mtime to record later // XXX: this will block; do we care? for (vector::iterator o = edge->outputs_.begin(); o != edge->outputs_.end(); ++o) { if (!disk_interface_->MakeDirs((*o)->path())) return false; + if (build_start == -1) { + disk_interface_->WriteFile(lock_file_path_, ""); + build_start = disk_interface_->Stat(lock_file_path_, err); + if (build_start == -1) + build_start = 0; + } } + edge->command_start_time_ = build_start; + + // Create depfile directory if needed. + // XXX: this may also block; do we care? + std::string depfile = edge->GetUnescapedDepfile(); + if (!depfile.empty() && !disk_interface_->MakeDirs(depfile)) + return false; + // Create response file, if needed // XXX: this may also block; do we care? string rspfile = edge->GetUnescapedRspfile(); @@ -950,9 +878,14 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { } } - int start_time, end_time; - status_->BuildEdgeFinished(edge, result->success(), result->output, - &start_time, &end_time); + int64_t start_time_millis, end_time_millis; + RunningEdgeMap::iterator it = running_edges_.find(edge); + start_time_millis = it->second; + end_time_millis = GetTimeMillis() - start_time_millis_; + running_edges_.erase(it); + + status_->BuildEdgeFinished(edge, start_time_millis, end_time_millis, + result->status, result->output); // The rest of this function only applies to successful commands. if (!result->success()) { @@ -960,55 +893,38 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { } // Restat the edge outputs - TimeStamp output_mtime = 0; - bool restat = edge->GetBindingBool("restat"); + TimeStamp record_mtime = 0; if (!config_.dry_run) { + const bool restat = edge->GetBindingBool("restat"); + const bool generator = edge->GetBindingBool("generator"); bool node_cleaned = false; - - for (vector::iterator o = edge->outputs_.begin(); - o != edge->outputs_.end(); ++o) { - TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), err); - if (new_mtime == -1) - return false; - if (new_mtime > output_mtime) - output_mtime = new_mtime; - if ((*o)->mtime() == new_mtime && restat) { - // The rule command did not change the output. Propagate the clean - // state through the build graph. - // Note that this also applies to nonexistent outputs (mtime == 0). - if (!plan_.CleanNode(&scan_, *o, err)) + record_mtime = edge->command_start_time_; + + // restat and generator rules must restat the outputs after the build + // has finished. if record_mtime == 0, then there was an error while + // attempting to touch/stat the temp file when the edge started and + // we should fall back to recording the outputs' current mtime in the + // log. + if (record_mtime == 0 || restat || generator) { + for (vector::iterator o = edge->outputs_.begin(); + o != edge->outputs_.end(); ++o) { + TimeStamp new_mtime = disk_interface_->Stat((*o)->path(), err); + if (new_mtime == -1) return false; - node_cleaned = true; + if (new_mtime > record_mtime) + record_mtime = new_mtime; + if ((*o)->mtime() == new_mtime && restat) { + // The rule command did not change the output. Propagate the clean + // state through the build graph. + // Note that this also applies to nonexistent outputs (mtime == 0). + if (!plan_.CleanNode(&scan_, *o, err)) + return false; + node_cleaned = true; + } } } - if (node_cleaned) { - TimeStamp restat_mtime = 0; - // If any output was cleaned, find the most recent mtime of any - // (existing) non-order-only input or the depfile. - for (vector::iterator i = edge->inputs_.begin(); - i != edge->inputs_.end() - edge->order_only_deps_; ++i) { - TimeStamp input_mtime = disk_interface_->Stat((*i)->path(), err); - if (input_mtime == -1) - return false; - if (input_mtime > restat_mtime) - restat_mtime = input_mtime; - } - - string depfile = edge->GetUnescapedDepfile(); - if (restat_mtime != 0 && deps_type.empty() && !depfile.empty()) { - TimeStamp depfile_mtime = disk_interface_->Stat(depfile, err); - if (depfile_mtime == -1) - return false; - if (depfile_mtime > restat_mtime) - restat_mtime = depfile_mtime; - } - - // The total number of edges in the plan may have changed as a result - // of a restat. - status_->PlanHasTotalEdges(plan_.command_edge_count()); - - output_mtime = restat_mtime; + record_mtime = edge->command_start_time_; } } @@ -1021,22 +937,24 @@ bool Builder::FinishCommand(CommandRunner::Result* result, string* err) { disk_interface_->RemoveFile(rspfile); if (scan_.build_log()) { - if (!scan_.build_log()->RecordCommand(edge, start_time, end_time, - output_mtime)) { + if (!scan_.build_log()->RecordCommand(edge, start_time_millis, + end_time_millis, record_mtime)) { *err = string("Error writing to build log: ") + strerror(errno); return false; } } if (!deps_type.empty() && !config_.dry_run) { - assert(edge->outputs_.size() == 1 && "should have been rejected by parser"); - Node* out = edge->outputs_[0]; - TimeStamp deps_mtime = disk_interface_->Stat(out->path(), err); - if (deps_mtime == -1) - return false; - if (!scan_.deps_log()->RecordDeps(out, deps_mtime, deps_nodes)) { - *err = string("Error writing to deps log: ") + strerror(errno); - return false; + assert(!edge->outputs_.empty() && "should have been rejected by parser"); + for (std::vector::const_iterator o = edge->outputs_.begin(); + o != edge->outputs_.end(); ++o) { + TimeStamp deps_mtime = disk_interface_->Stat((*o)->path(), err); + if (deps_mtime == -1) + return false; + if (!scan_.deps_log()->RecordDeps(*o, deps_mtime, deps_nodes)) { + *err = std::string("Error writing to deps log: ") + strerror(errno); + return false; + } } } return true; @@ -1061,8 +979,7 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, // complexity in IncludesNormalize::Relativize. deps_nodes->push_back(state_->GetNode(*i, ~0u)); } - } else - if (deps_type == "gcc") { + } else if (deps_type == "gcc") { string depfile = result->edge->GetUnescapedDepfile(); if (depfile.empty()) { *err = string("edge with deps=gcc but no depfile makes no sense"); @@ -1092,9 +1009,7 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, for (vector::iterator i = deps.ins_.begin(); i != deps.ins_.end(); ++i) { uint64_t slash_bits; - if (!CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits, - err)) - return false; + CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits); deps_nodes->push_back(state_->GetNode(*i, slash_bits)); } @@ -1112,8 +1027,6 @@ bool Builder::ExtractDeps(CommandRunner::Result* result, } bool Builder::LoadDyndeps(Node* node, string* err) { - status_->BuildLoadDyndeps(); - // Load the dyndep information provided by this node. DyndepFile ddf; if (!scan_.LoadDyndeps(node, &ddf, err)) @@ -1123,8 +1036,12 @@ bool Builder::LoadDyndeps(Node* node, string* err) { if (!plan_.DyndepsLoaded(&scan_, node, ddf, err)) return false; - // New command edges may have been added to the plan. - status_->PlanHasTotalEdges(plan_.command_edge_count()); - return true; } + +void Builder::SetFailureCode(ExitStatus code) { + // ExitSuccess should not overwrite any error + if (code != ExitSuccess) { + exit_code_ = code; + } +} diff --git a/src/build.h b/src/build.h index ab59f0c096..04e007627f 100644 --- a/src/build.h +++ b/src/build.h @@ -18,25 +18,22 @@ #include #include #include -#include -#include #include #include #include "depfile_parser.h" -#include "graph.h" // XXX needed for DependencyScan; should rearrange. #include "exit_status.h" -#include "line_printer.h" -#include "metrics.h" +#include "graph.h" #include "util.h" // int64_t struct BuildLog; -struct BuildStatus; struct Builder; struct DiskInterface; struct Edge; +struct Explanations; struct Node; struct State; +struct Status; /// Plan stores the state of a build plan: what we intend to build, /// which steps we're ready to execute. @@ -46,7 +43,7 @@ struct Plan { /// Add a target to our plan (including all its dependencies). /// Returns false if we don't need to build this target; may /// fill in |err| with an error message if there's a problem. - bool AddTarget(Node* node, string* err); + bool AddTarget(const Node* target, std::string* err); // Pop a ready edge off the queue of edges to build. // Returns NULL if there's no work to do. @@ -56,7 +53,7 @@ struct Plan { bool more_to_do() const { return wanted_edges_ > 0 && command_edges_ > 0; } /// Dumps the current state of the plan. - void Dump(); + void Dump() const; enum EdgeResult { kEdgeFailed, @@ -67,11 +64,11 @@ struct Plan { /// If any of the edge's outputs are dyndep bindings of their dependents, /// this loads dynamic dependencies from the nodes' paths. /// Returns 'false' if loading dyndep info fails and 'true' otherwise. - bool EdgeFinished(Edge* edge, EdgeResult result, string* err); + bool EdgeFinished(Edge* edge, EdgeResult result, std::string* err); /// Clean the given node during the build. /// Return false on error. - bool CleanNode(DependencyScan* scan, Node* node, string* err); + bool CleanNode(DependencyScan* scan, Node* node, std::string* err); /// Number of edges with commands to run. int command_edge_count() const { return command_edges_; } @@ -79,21 +76,13 @@ struct Plan { /// Reset state. Clears want and ready sets. void Reset(); + // After all targets have been added, prepares the ready queue for find work. + void PrepareQueue(); + /// Update the build plan to account for modifications made to the graph /// by information loaded from a dyndep file. - bool DyndepsLoaded(DependencyScan* scan, Node* node, - const DyndepFile& ddf, string* err); -private: - bool RefreshDyndepDependents(DependencyScan* scan, Node* node, string* err); - void UnmarkDependents(Node* node, set* dependents); - bool AddSubTarget(Node* node, Node* dependent, string* err, - set* dyndep_walk); - - /// Update plan with knowledge that the given node is up to date. - /// If the node is a dyndep binding on any of its dependents, this - /// loads dynamic dependencies from the node's path. - /// Returns 'false' if loading dyndep info fails and 'true' otherwise. - bool NodeFinished(Node* node, string* err); + bool DyndepsLoaded(DependencyScan* scan, const Node* node, + const DyndepFile& ddf, std::string* err); /// Enumerate possible steps we want for an edge. enum Want @@ -108,23 +97,42 @@ struct Plan { kWantToFinish }; - void EdgeWanted(Edge* edge); - bool EdgeMaybeReady(map::iterator want_e, string* err); +private: + void ComputeCriticalPath(); + bool RefreshDyndepDependents(DependencyScan* scan, const Node* node, std::string* err); + void UnmarkDependents(const Node* node, std::set* dependents); + bool AddSubTarget(const Node* node, const Node* dependent, std::string* err, + std::set* dyndep_walk); + + // Add edges that kWantToStart into the ready queue + // Must be called after ComputeCriticalPath and before FindWork + void ScheduleInitialEdges(); + + /// Update plan with knowledge that the given node is up to date. + /// If the node is a dyndep binding on any of its dependents, this + /// loads dynamic dependencies from the node's path. + /// Returns 'false' if loading dyndep info fails and 'true' otherwise. + bool NodeFinished(Node* node, std::string* err); + + void EdgeWanted(const Edge* edge); + bool EdgeMaybeReady(std::map::iterator want_e, std::string* err); /// Submits a ready edge as a candidate for execution. /// The edge may be delayed from running, for example if it's a member of a /// currently-full pool. - void ScheduleWork(map::iterator want_e); + void ScheduleWork(std::map::iterator want_e); /// Keep track of which edges we want to build in this plan. If this map does /// not contain an entry for an edge, we do not want to build the entry or its /// dependents. If it does contain an entry, the enumeration indicates what /// we want for the edge. - map want_; + std::map want_; - set ready_; + EdgePriorityQueue ready_; Builder* builder_; + /// user provided targets in build order, earlier one have higher priority + std::vector targets_; /// Total number of edges that have commands (not phony). int command_edges_; @@ -133,12 +141,14 @@ struct Plan { int wanted_edges_; }; +struct BuildConfig; + /// CommandRunner is an interface that wraps running the build /// subcommands. This allows tests to abstract out running commands. /// RealCommandRunner is an implementation that actually runs commands. struct CommandRunner { virtual ~CommandRunner() {} - virtual bool CanRunMore() = 0; + virtual size_t CanRunMore() const = 0; virtual bool StartCommand(Edge* edge) = 0; /// The result of waiting for a command. @@ -146,14 +156,17 @@ struct CommandRunner { Result() : edge(NULL) {} Edge* edge; ExitStatus status; - string output; + std::string output; bool success() const { return status == ExitSuccess; } }; /// Wait for a command to complete, or return false if interrupted. virtual bool WaitForCommand(Result* result) = 0; - virtual vector GetActiveEdges() { return vector(); } + virtual std::vector GetActiveEdges() { return std::vector(); } virtual void Abort() {} + + /// Creates the RealCommandRunner + static CommandRunner* factory(const BuildConfig& config); }; /// Options (e.g. verbosity, parallelism) passed to a build. @@ -162,8 +175,9 @@ struct BuildConfig { failures_allowed(1), max_load_average(-0.0f) {} enum Verbosity { - NORMAL, QUIET, // No output -- used when testing. + NO_STATUS_UPDATE, // just regular output but suppress status update + NORMAL, // regular output and status update VERBOSE }; Verbosity verbosity; @@ -178,32 +192,32 @@ struct BuildConfig { /// Builder wraps the build process: starting commands, updating status. struct Builder { - Builder(State* state, const BuildConfig& config, - BuildLog* build_log, DepsLog* deps_log, - DiskInterface* disk_interface); + Builder(State* state, const BuildConfig& config, BuildLog* build_log, + DepsLog* deps_log, DiskInterface* disk_interface, Status* status, + int64_t start_time_millis); ~Builder(); /// Clean up after interrupted commands by deleting output files. void Cleanup(); - Node* AddTarget(const string& name, string* err); + Node* AddTarget(const std::string& name, std::string* err); /// Add a target to the build, scanning dependencies. /// @return false on error. - bool AddTarget(Node* target, string* err); + bool AddTarget(Node* target, std::string* err); /// Returns true if the build targets are already up to date. bool AlreadyUpToDate() const; - /// Run the build. Returns false on error. + /// Run the build. Returns ExitStatus or the exit code of the last failed job. /// It is an error to call this function when AlreadyUpToDate() is true. - bool Build(string* err); + ExitStatus Build(std::string* err); - bool StartEdge(Edge* edge, string* err); + bool StartEdge(Edge* edge, std::string* err); /// Update status ninja logs following a command termination. /// @return false if the build can not proceed further due to a fatal error. - bool FinishCommand(CommandRunner::Result* result, string* err); + bool FinishCommand(CommandRunner::Result* result, std::string* err); /// Used for tests. void SetBuildLog(BuildLog* log) { @@ -211,128 +225,45 @@ struct Builder { } /// Load the dyndep information provided by the given node. - bool LoadDyndeps(Node* node, string* err); + bool LoadDyndeps(Node* node, std::string* err); State* state_; const BuildConfig& config_; Plan plan_; -#if __cplusplus < 201703L - auto_ptr command_runner_; -#else - unique_ptr command_runner_; // auto_ptr was removed in C++17. -#endif - BuildStatus* status_; - - private: - bool ExtractDeps(CommandRunner::Result* result, const string& deps_type, - const string& deps_prefix, vector* deps_nodes, - string* err); - - DiskInterface* disk_interface_; - DependencyScan scan_; - - // Unimplemented copy ctor and operator= ensure we don't copy the auto_ptr. - Builder(const Builder &other); // DO NOT IMPLEMENT - void operator=(const Builder &other); // DO NOT IMPLEMENT -}; + std::unique_ptr command_runner_; + Status* status_; -/// Tracks the status of a build: completion fraction, printing updates. -struct BuildStatus { - explicit BuildStatus(const BuildConfig& config); - void PlanHasTotalEdges(int total); - void BuildEdgeStarted(Edge* edge); - void BuildEdgeFinished(Edge* edge, bool success, const string& output, - int* start_time, int* end_time); - void BuildLoadDyndeps(); - void BuildStarted(); - void BuildFinished(); - - enum EdgeStatus { - kEdgeStarted, - kEdgeFinished, - }; - - /// Format the progress status string by replacing the placeholders. - /// See the user manual for more information about the available - /// placeholders. - /// @param progress_status_format The format of the progress status. - /// @param status The status of the edge. - string FormatProgressStatus(const char* progress_status_format, - EdgeStatus status) const; + /// Returns ExitStatus or the exit code of the last failed job + /// (doesn't need to be an enum value of ExitStatus) + ExitStatus GetExitCode() const { return exit_code_; } private: - void PrintStatus(Edge* edge, EdgeStatus status); - - const BuildConfig& config_; - - /// Time the build started. - int64_t start_time_millis_; - - int started_edges_, finished_edges_, total_edges_; + bool ExtractDeps(CommandRunner::Result* result, const std::string& deps_type, + const std::string& deps_prefix, + std::vector* deps_nodes, std::string* err); /// Map of running edge to time the edge started running. - typedef map RunningEdgeMap; + typedef std::map RunningEdgeMap; RunningEdgeMap running_edges_; - /// Prints progress output. - LinePrinter printer_; - - /// The custom progress status format to use. - const char* progress_status_format_; - - template - void SnprintfRate(double rate, char(&buf)[S], const char* format) const { - if (rate == -1) - snprintf(buf, S, "?"); - else - snprintf(buf, S, format, rate); - } - - struct RateInfo { - RateInfo() : rate_(-1) {} + /// Time the build started. + int64_t start_time_millis_; - void Restart() { stopwatch_.Restart(); } - double Elapsed() const { return stopwatch_.Elapsed(); } - double rate() { return rate_; } + std::string lock_file_path_; + DiskInterface* disk_interface_; - void UpdateRate(int edges) { - if (edges && stopwatch_.Elapsed()) - rate_ = edges / stopwatch_.Elapsed(); - } + // Only create an Explanations class if '-d explain' is used. + std::unique_ptr explanations_; - private: - double rate_; - Stopwatch stopwatch_; - }; + DependencyScan scan_; - struct SlidingRateInfo { - SlidingRateInfo(int n) : rate_(-1), N(n), last_update_(-1) {} - - void Restart() { stopwatch_.Restart(); } - double rate() { return rate_; } - - void UpdateRate(int update_hint) { - if (update_hint == last_update_) - return; - last_update_ = update_hint; - - if (times_.size() == N) - times_.pop(); - times_.push(stopwatch_.Elapsed()); - if (times_.back() != times_.front()) - rate_ = times_.size() / (times_.back() - times_.front()); - } - - private: - double rate_; - Stopwatch stopwatch_; - const size_t N; - queue times_; - int last_update_; - }; + /// Keep the global exit code for the build + ExitStatus exit_code_ = ExitSuccess; + void SetFailureCode(ExitStatus code); - mutable RateInfo overall_rate_; - mutable SlidingRateInfo current_rate_; + // Unimplemented copy ctor and operator= ensure we don't copy the auto_ptr. + Builder(const Builder &other); // DO NOT IMPLEMENT + void operator=(const Builder &other); // DO NOT IMPLEMENT }; #endif // NINJA_BUILD_H_ diff --git a/src/build_log.cc b/src/build_log.cc index 774f72f878..890a3cd3b3 100644 --- a/src/build_log.cc +++ b/src/build_log.cc @@ -21,7 +21,9 @@ #endif #include "build_log.h" +#include "disk_interface.h" +#include #include #include #include @@ -49,132 +51,66 @@ namespace { const char kFileSignature[] = "# ninja log v%d\n"; -const char kFileColumnLabels[] = "# start_time end_time mtime command hash\n"; -const int kOldestSupportedVersion = 4; -const int kCurrentVersion = 5; - -// 64bit MurmurHash2, by Austin Appleby -#if defined(_MSC_VER) -#define BIG_CONSTANT(x) (x) -#else // defined(_MSC_VER) -#define BIG_CONSTANT(x) (x##LLU) -#endif // !defined(_MSC_VER) -inline -uint64_t MurmurHash64A(const void* key, size_t len) { - static const uint64_t seed = 0xDECAFBADDECAFBADull; - const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995); - const int r = 47; - uint64_t h = seed ^ (len * m); - const unsigned char* data = (const unsigned char*)key; - while (len >= 8) { - uint64_t k; - memcpy(&k, data, sizeof k); - k *= m; - k ^= k >> r; - k *= m; - h ^= k; - h *= m; - data += 8; - len -= 8; - } - switch (len & 7) - { - case 7: h ^= uint64_t(data[6]) << 48; - NINJA_FALLTHROUGH; - case 6: h ^= uint64_t(data[5]) << 40; - NINJA_FALLTHROUGH; - case 5: h ^= uint64_t(data[4]) << 32; - NINJA_FALLTHROUGH; - case 4: h ^= uint64_t(data[3]) << 24; - NINJA_FALLTHROUGH; - case 3: h ^= uint64_t(data[2]) << 16; - NINJA_FALLTHROUGH; - case 2: h ^= uint64_t(data[1]) << 8; - NINJA_FALLTHROUGH; - case 1: h ^= uint64_t(data[0]); - h *= m; - }; - h ^= h >> r; - h *= m; - h ^= h >> r; - return h; -} -#undef BIG_CONSTANT - +const int kOldestSupportedVersion = 7; +const int kCurrentVersion = 7; } // namespace // static uint64_t BuildLog::LogEntry::HashCommand(StringPiece command) { - return MurmurHash64A(command.str_, command.len_); + return rapidhash(command.str_, command.len_); } -BuildLog::LogEntry::LogEntry(const string& output) - : output(output) {} +BuildLog::LogEntry::LogEntry(std::string output) : output(std::move(output)) {} -BuildLog::LogEntry::LogEntry(const string& output, uint64_t command_hash, - int start_time, int end_time, TimeStamp restat_mtime) - : output(output), command_hash(command_hash), - start_time(start_time), end_time(end_time), mtime(restat_mtime) -{} +BuildLog::LogEntry::LogEntry(const std::string& output, uint64_t command_hash, + int start_time, int end_time, TimeStamp mtime) + : output(output), command_hash(command_hash), start_time(start_time), + end_time(end_time), mtime(mtime) {} -BuildLog::BuildLog() - : log_file_(NULL), needs_recompaction_(false) {} +BuildLog::BuildLog() = default; BuildLog::~BuildLog() { Close(); } -bool BuildLog::OpenForWrite(const string& path, const BuildLogUser& user, - string* err) { +bool BuildLog::OpenForWrite(const std::string& path, const BuildLogUser& user, + std::string* err) { if (needs_recompaction_) { if (!Recompact(path, user, err)) return false; } - log_file_ = fopen(path.c_str(), "ab"); - if (!log_file_) { - *err = strerror(errno); - return false; - } - setvbuf(log_file_, NULL, _IOLBF, BUFSIZ); - SetCloseOnExec(fileno(log_file_)); - - // Opening a file in append mode doesn't set the file pointer to the file's - // end on Windows. Do that explicitly. - fseek(log_file_, 0, SEEK_END); - - if (ftell(log_file_) == 0) { - if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0 || - fprintf(log_file_, kFileColumnLabels) < 0) { - *err = strerror(errno); - return false; - } - } - + assert(!log_file_); + log_file_path_ = path; // we don't actually open the file right now, but will + // do so on the first write attempt return true; } bool BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, TimeStamp mtime) { - string command = edge->EvaluateCommand(true); + std::string command = edge->EvaluateCommand(true); uint64_t command_hash = LogEntry::HashCommand(command); - for (vector::iterator out = edge->outputs_.begin(); + for (std::vector::iterator out = edge->outputs_.begin(); out != edge->outputs_.end(); ++out) { - const string& path = (*out)->path(); + const std::string& path = (*out)->path(); Entries::iterator i = entries_.find(path); LogEntry* log_entry; if (i != entries_.end()) { - log_entry = i->second; + log_entry = i->second.get(); } else { log_entry = new LogEntry(path); - entries_.insert(Entries::value_type(log_entry->output, log_entry)); + // Passes ownership of |log_entry| to the map, but keeps the pointer valid. + entries_.emplace(log_entry->output, std::unique_ptr(log_entry)); } log_entry->command_hash = command_hash; log_entry->start_time = start_time; log_entry->end_time = end_time; log_entry->mtime = mtime; + if (!OpenForWriteIfNeeded()) { + return false; + } if (log_file_) { if (!WriteEntry(log_file_, *log_entry)) return false; @@ -187,11 +123,37 @@ bool BuildLog::RecordCommand(Edge* edge, int start_time, int end_time, } void BuildLog::Close() { + OpenForWriteIfNeeded(); // create the file even if nothing has been recorded if (log_file_) fclose(log_file_); log_file_ = NULL; } +bool BuildLog::OpenForWriteIfNeeded() { + if (log_file_ || log_file_path_.empty()) { + return true; + } + log_file_ = fopen(log_file_path_.c_str(), "ab"); + if (!log_file_) { + return false; + } + if (setvbuf(log_file_, NULL, _IOLBF, BUFSIZ) != 0) { + return false; + } + SetCloseOnExec(fileno(log_file_)); + + // Opening a file in append mode doesn't set the file pointer to the file's + // end on Windows. Do that explicitly. + fseek(log_file_, 0, SEEK_END); + + if (ftell(log_file_) == 0) { + if (fprintf(log_file_, kFileSignature, kCurrentVersion) < 0) { + return false; + } + } + return true; +} + struct LineReader { explicit LineReader(FILE* file) : file_(file), buf_end_(buf_), line_start_(buf_), line_end_(NULL) { @@ -215,7 +177,7 @@ struct LineReader { line_start_ = line_end_ + 1; } - line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_); + line_end_ = static_cast(memchr(line_start_, '\n', buf_end_ - line_start_)); if (!line_end_) { // No newline. Move rest of data to start of buffer, fill rest. size_t already_consumed = line_start_ - buf_; @@ -225,7 +187,7 @@ struct LineReader { size_t read = fread(buf_ + size_rest, 1, sizeof(buf_) - size_rest, file_); buf_end_ = buf_ + size_rest + read; line_start_ = buf_; - line_end_ = (char*)memchr(line_start_, '\n', buf_end_ - line_start_); + line_end_ = static_cast(memchr(line_start_, '\n', buf_end_ - line_start_)); } *line_start = line_start_; @@ -243,14 +205,14 @@ struct LineReader { char* line_end_; }; -bool BuildLog::Load(const string& path, string* err) { +LoadStatus BuildLog::Load(const std::string& path, std::string* err) { METRIC_RECORD(".ninja_log load"); FILE* file = fopen(path.c_str(), "r"); if (!file) { if (errno == ENOENT) - return true; + return LOAD_NOT_FOUND; *err = strerror(errno); - return false; + return LOAD_ERROR; } int log_version = 0; @@ -264,14 +226,21 @@ bool BuildLog::Load(const string& path, string* err) { if (!log_version) { sscanf(line_start, kFileSignature, &log_version); + bool invalid_log_version = false; if (log_version < kOldestSupportedVersion) { - *err = ("build log version invalid, perhaps due to being too old; " - "starting over"); + invalid_log_version = true; + *err = "build log version is too old; starting over"; + + } else if (log_version > kCurrentVersion) { + invalid_log_version = true; + *err = "build log version is too new; starting over"; + } + if (invalid_log_version) { fclose(file); - unlink(path.c_str()); - // Don't report this as a failure. An empty build log will cause + platformAwareUnlink(path.c_str()); + // Don't report this as a failure. A missing build log will cause // us to rebuild the outputs anyway. - return true; + return LOAD_NOT_FOUND; } } @@ -282,35 +251,35 @@ bool BuildLog::Load(const string& path, string* err) { const char kFieldSeparator = '\t'; char* start = line_start; - char* end = (char*)memchr(start, kFieldSeparator, line_end - start); + char* end = static_cast(memchr(start, kFieldSeparator, line_end - start)); if (!end) continue; *end = 0; int start_time = 0, end_time = 0; - TimeStamp restat_mtime = 0; + TimeStamp mtime = 0; start_time = atoi(start); start = end + 1; - end = (char*)memchr(start, kFieldSeparator, line_end - start); + end = static_cast(memchr(start, kFieldSeparator, line_end - start)); if (!end) continue; *end = 0; end_time = atoi(start); start = end + 1; - end = (char*)memchr(start, kFieldSeparator, line_end - start); + end = static_cast(memchr(start, kFieldSeparator, line_end - start)); if (!end) continue; *end = 0; - restat_mtime = strtoll(start, NULL, 10); + mtime = strtoll(start, NULL, 10); start = end + 1; - end = (char*)memchr(start, kFieldSeparator, line_end - start); + end = static_cast(memchr(start, kFieldSeparator, line_end - start)); if (!end) continue; - string output = string(start, end - start); + std::string output(start, end - start); start = end + 1; end = line_end; @@ -318,30 +287,26 @@ bool BuildLog::Load(const string& path, string* err) { LogEntry* entry; Entries::iterator i = entries_.find(output); if (i != entries_.end()) { - entry = i->second; + entry = i->second.get(); } else { - entry = new LogEntry(output); - entries_.insert(Entries::value_type(entry->output, entry)); + entry = new LogEntry(std::move(output)); + // Passes ownership of |entry| to the map, but keeps the pointer valid. + entries_.emplace(entry->output, std::unique_ptr(entry)); ++unique_entry_count; } ++total_entry_count; entry->start_time = start_time; entry->end_time = end_time; - entry->mtime = restat_mtime; - if (log_version >= 5) { - char c = *end; *end = '\0'; - entry->command_hash = (uint64_t)strtoull(start, NULL, 16); - *end = c; - } else { - entry->command_hash = LogEntry::HashCommand(StringPiece(start, - end - start)); - } + entry->mtime = mtime; + char c = *end; *end = '\0'; + entry->command_hash = (uint64_t)strtoull(start, NULL, 16); + *end = c; } fclose(file); if (!line_start) { - return true; // file was empty + return LOAD_SUCCESS; // file was empty } // Decide whether it's time to rebuild the log: @@ -356,13 +321,13 @@ bool BuildLog::Load(const string& path, string* err) { needs_recompaction_ = true; } - return true; + return LOAD_SUCCESS; } -BuildLog::LogEntry* BuildLog::LookupByOutput(const string& path) { +BuildLog::LogEntry* BuildLog::LookupByOutput(const std::string& path) { Entries::iterator i = entries_.find(path); if (i != entries_.end()) - return i->second; + return i->second.get(); return NULL; } @@ -372,12 +337,12 @@ bool BuildLog::WriteEntry(FILE* f, const LogEntry& entry) { entry.output.c_str(), entry.command_hash) > 0; } -bool BuildLog::Recompact(const string& path, const BuildLogUser& user, - string* err) { +bool BuildLog::Recompact(const std::string& path, const BuildLogUser& user, + std::string* err) { METRIC_RECORD(".ninja_log recompact"); Close(); - string temp_path = path + ".recompact"; + std::string temp_path = path + ".recompact"; FILE* f = fopen(temp_path.c_str(), "wb"); if (!f) { *err = strerror(errno); @@ -390,25 +355,25 @@ bool BuildLog::Recompact(const string& path, const BuildLogUser& user, return false; } - vector dead_outputs; - for (Entries::iterator i = entries_.begin(); i != entries_.end(); ++i) { - if (user.IsPathDead(i->first)) { - dead_outputs.push_back(i->first); + std::vector dead_outputs; + for (const auto& pair : entries_) { + if (user.IsPathDead(pair.first)) { + dead_outputs.push_back(pair.first); continue; } - if (!WriteEntry(f, *i->second)) { + if (!WriteEntry(f, *pair.second)) { *err = strerror(errno); fclose(f); return false; } } - for (size_t i = 0; i < dead_outputs.size(); ++i) - entries_.erase(dead_outputs[i]); + for (StringPiece output : dead_outputs) + entries_.erase(output); fclose(f); - if (unlink(path.c_str()) < 0) { + if (platformAwareUnlink(path.c_str()) < 0) { *err = strerror(errno); return false; } @@ -420,3 +385,60 @@ bool BuildLog::Recompact(const string& path, const BuildLogUser& user, return true; } + +bool BuildLog::Restat(const StringPiece path, + const DiskInterface& disk_interface, + const int output_count, char** outputs, + std::string* const err) { + METRIC_RECORD(".ninja_log restat"); + + Close(); + std::string temp_path = path.AsString() + ".restat"; + FILE* f = fopen(temp_path.c_str(), "wb"); + if (!f) { + *err = strerror(errno); + return false; + } + + if (fprintf(f, kFileSignature, kCurrentVersion) < 0) { + *err = strerror(errno); + fclose(f); + return false; + } + for (auto& pair : entries_) { + bool skip = output_count > 0; + for (int j = 0; j < output_count; ++j) { + if (pair.second->output == outputs[j]) { + skip = false; + break; + } + } + if (!skip) { + const TimeStamp mtime = disk_interface.Stat(pair.second->output, err); + if (mtime == -1) { + fclose(f); + return false; + } + pair.second->mtime = mtime; + } + + if (!WriteEntry(f, *pair.second)) { + *err = strerror(errno); + fclose(f); + return false; + } + } + + fclose(f); + if (platformAwareUnlink(path.str_) < 0) { + *err = strerror(errno); + return false; + } + + if (rename(temp_path.c_str(), path.str_) < 0) { + *err = strerror(errno); + return false; + } + + return true; +} diff --git a/src/build_log.h b/src/build_log.h index 5268fabb68..61bd8ffbf2 100644 --- a/src/build_log.h +++ b/src/build_log.h @@ -15,14 +15,17 @@ #ifndef NINJA_BUILD_LOG_H_ #define NINJA_BUILD_LOG_H_ -#include #include -using namespace std; + +#include +#include #include "hash_map.h" +#include "load_status.h" #include "timestamp.h" #include "util.h" // uint64_t +struct DiskInterface; struct Edge; /// Can answer questions about the manifest for the BuildLog. @@ -43,51 +46,64 @@ struct BuildLog { BuildLog(); ~BuildLog(); - bool OpenForWrite(const string& path, const BuildLogUser& user, string* err); + /// Prepares writing to the log file without actually opening it - that will + /// happen when/if it's needed + bool OpenForWrite(const std::string& path, const BuildLogUser& user, + std::string* err); bool RecordCommand(Edge* edge, int start_time, int end_time, TimeStamp mtime = 0); void Close(); /// Load the on-disk log. - bool Load(const string& path, string* err); + LoadStatus Load(const std::string& path, std::string* err); struct LogEntry { - string output; - uint64_t command_hash; - int start_time; - int end_time; - TimeStamp mtime; + std::string output; + uint64_t command_hash = 0; + int start_time = 0; + int end_time = 0; + TimeStamp mtime = 0; static uint64_t HashCommand(StringPiece command); // Used by tests. - bool operator==(const LogEntry& o) { + bool operator==(const LogEntry& o) const { return output == o.output && command_hash == o.command_hash && start_time == o.start_time && end_time == o.end_time && mtime == o.mtime; } - explicit LogEntry(const string& output); - LogEntry(const string& output, uint64_t command_hash, - int start_time, int end_time, TimeStamp restat_mtime); + explicit LogEntry(std::string output); + LogEntry(const std::string& output, uint64_t command_hash, + int start_time, int end_time, TimeStamp mtime); }; /// Lookup a previously-run command by its output path. - LogEntry* LookupByOutput(const string& path); + LogEntry* LookupByOutput(const std::string& path); /// Serialize an entry into a log file. bool WriteEntry(FILE* f, const LogEntry& entry); /// Rewrite the known log entries, throwing away old data. - bool Recompact(const string& path, const BuildLogUser& user, string* err); + bool Recompact(const std::string& path, const BuildLogUser& user, + std::string* err); - typedef ExternalStringHashMap::Type Entries; + /// Restat all outputs in the log + bool Restat(StringPiece path, const DiskInterface& disk_interface, + int output_count, char** outputs, std::string* err); + + typedef ExternalStringHashMap>::Type Entries; const Entries& entries() const { return entries_; } private: + /// Should be called before using log_file_. When false is returned, errno + /// will be set. + bool OpenForWriteIfNeeded(); + Entries entries_; - FILE* log_file_; - bool needs_recompaction_; + FILE* log_file_ = nullptr; + std::string log_file_path_; + bool needs_recompaction_ = false; }; #endif // NINJA_BUILD_LOG_H_ diff --git a/src/build_log_perftest.cc b/src/build_log_perftest.cc index e471d138cc..869112f080 100644 --- a/src/build_log_perftest.cc +++ b/src/build_log_perftest.cc @@ -26,6 +26,8 @@ #include #endif +using namespace std; + const char kTestFilename[] = "BuildLogPerfTest-tempfile"; struct NoDeadPaths : public BuildLogUser { @@ -110,7 +112,7 @@ int main() { { // Read once to warm up disk cache. BuildLog log; - if (!log.Load(kTestFilename, &err)) { + if (log.Load(kTestFilename, &err) == LOAD_ERROR) { fprintf(stderr, "Failed to read test data: %s\n", err.c_str()); return 1; } @@ -119,7 +121,7 @@ int main() { for (int i = 0; i < kNumRepetitions; ++i) { int64_t start = GetTimeMillis(); BuildLog log; - if (!log.Load(kTestFilename, &err)) { + if (log.Load(kTestFilename, &err) == LOAD_ERROR) { fprintf(stderr, "Failed to read test data: %s\n", err.c_str()); return 1; } @@ -142,8 +144,7 @@ int main() { printf("min %dms max %dms avg %.1fms\n", min, max, total / times.size()); - unlink(kTestFilename); + platformAwareUnlink(kTestFilename); return 0; } - diff --git a/src/build_log_test.cc b/src/build_log_test.cc index eea818f961..47de8b5d0c 100644 --- a/src/build_log_test.cc +++ b/src/build_log_test.cc @@ -25,6 +25,7 @@ #include #include #endif +#include namespace { @@ -33,10 +34,10 @@ const char kTestFilename[] = "BuildLogTest-tempfile"; struct BuildLogTest : public StateTestWithBuiltinRules, public BuildLogUser { virtual void SetUp() { // In case a crashing test left a stale file behind. - unlink(kTestFilename); + platformAwareUnlink(kTestFilename); } virtual void TearDown() { - unlink(kTestFilename); + platformAwareUnlink(kTestFilename); } virtual bool IsPathDead(StringPiece s) const { return false; } }; @@ -47,7 +48,7 @@ TEST_F(BuildLogTest, WriteRead) { "build mid: cat in\n"); BuildLog log1; - string err; + std::string err; EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err)); ASSERT_EQ("", err); log1.RecordCommand(state_.edges_[0], 15, 18); @@ -70,12 +71,11 @@ TEST_F(BuildLogTest, WriteRead) { } TEST_F(BuildLogTest, FirstWriteAddsSignature) { - const char kExpectedContent[] = "# ninja log vX\n" - "# start_time end_time mtime command hash\n"; - const size_t kVersionPos = 13; // Points at 'X'. + const char kExpectedVersion[] = "# ninja log vX\n"; + const size_t kVersionPos = strlen(kExpectedVersion) - 2; // Points at 'X'. BuildLog log; - string contents, err; + std::string contents, err; EXPECT_TRUE(log.OpenForWrite(kTestFilename, *this, &err)); ASSERT_EQ("", err); @@ -85,7 +85,7 @@ TEST_F(BuildLogTest, FirstWriteAddsSignature) { ASSERT_EQ("", err); if (contents.size() >= kVersionPos) contents[kVersionPos] = 'X'; - EXPECT_EQ(kExpectedContent, contents); + EXPECT_EQ(kExpectedVersion, contents); // Opening the file anew shouldn't add a second version string. EXPECT_TRUE(log.OpenForWrite(kTestFilename, *this, &err)); @@ -97,17 +97,19 @@ TEST_F(BuildLogTest, FirstWriteAddsSignature) { ASSERT_EQ("", err); if (contents.size() >= kVersionPos) contents[kVersionPos] = 'X'; - EXPECT_EQ(kExpectedContent, contents); + EXPECT_EQ(kExpectedVersion, contents); } TEST_F(BuildLogTest, DoubleEntry) { FILE* f = fopen(kTestFilename, "wb"); - fprintf(f, "# ninja log v4\n"); - fprintf(f, "0\t1\t2\tout\tcommand abc\n"); - fprintf(f, "3\t4\t5\tout\tcommand def\n"); + fprintf(f, "# ninja log v7\n"); + fprintf(f, "0\t1\t2\tout\t%" PRIx64 "\n", + BuildLog::LogEntry::HashCommand("command abc")); + fprintf(f, "0\t1\t2\tout\t%" PRIx64 "\n", + BuildLog::LogEntry::HashCommand("command def")); fclose(f); - string err; + std::string err; BuildLog log; EXPECT_TRUE(log.Load(kTestFilename, &err)); ASSERT_EQ("", err); @@ -124,23 +126,27 @@ TEST_F(BuildLogTest, Truncate) { { BuildLog log1; - string err; + std::string err; EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err)); ASSERT_EQ("", err); log1.RecordCommand(state_.edges_[0], 15, 18); log1.RecordCommand(state_.edges_[1], 20, 25); log1.Close(); } - +#ifdef __USE_LARGEFILE64 + struct stat64 statbuf; + ASSERT_EQ(0, stat64(kTestFilename, &statbuf)); +#else struct stat statbuf; ASSERT_EQ(0, stat(kTestFilename, &statbuf)); +#endif ASSERT_GT(statbuf.st_size, 0); // For all possible truncations of the input file, assert that we don't // crash when parsing. for (off_t size = statbuf.st_size; size > 0; --size) { BuildLog log2; - string err; + std::string err; EXPECT_TRUE(log2.OpenForWrite(kTestFilename, *this, &err)); ASSERT_EQ("", err); log2.RecordCommand(state_.edges_[0], 15, 18); @@ -151,7 +157,7 @@ TEST_F(BuildLogTest, Truncate) { BuildLog log3; err.clear(); - ASSERT_TRUE(log3.Load(kTestFilename, &err) || !err.empty()); + ASSERT_TRUE(log3.Load(kTestFilename, &err) == LOAD_SUCCESS || !err.empty()); } } @@ -161,19 +167,20 @@ TEST_F(BuildLogTest, ObsoleteOldVersion) { fprintf(f, "123 456 0 out command\n"); fclose(f); - string err; + std::string err; BuildLog log; EXPECT_TRUE(log.Load(kTestFilename, &err)); - ASSERT_NE(err.find("version"), string::npos); + ASSERT_NE(err.find("version"), std::string::npos); } -TEST_F(BuildLogTest, SpacesInOutputV4) { +TEST_F(BuildLogTest, SpacesInOutput) { FILE* f = fopen(kTestFilename, "wb"); - fprintf(f, "# ninja log v4\n"); - fprintf(f, "123\t456\t456\tout with space\tcommand\n"); + fprintf(f, "# ninja log v7\n"); + fprintf(f, "123\t456\t456\tout with space\t%" PRIx64 "\n", + BuildLog::LogEntry::HashCommand("command")); fclose(f); - string err; + std::string err; BuildLog log; EXPECT_TRUE(log.Load(kTestFilename, &err)); ASSERT_EQ("", err); @@ -191,13 +198,15 @@ TEST_F(BuildLogTest, DuplicateVersionHeader) { // build log on Windows. This shouldn't crash, and the second version header // should be ignored. FILE* f = fopen(kTestFilename, "wb"); - fprintf(f, "# ninja log v4\n"); - fprintf(f, "123\t456\t456\tout\tcommand\n"); - fprintf(f, "# ninja log v4\n"); - fprintf(f, "456\t789\t789\tout2\tcommand2\n"); + fprintf(f, "# ninja log v7\n"); + fprintf(f, "123\t456\t456\tout\t%" PRIx64 "\n", + BuildLog::LogEntry::HashCommand("command")); + fprintf(f, "# ninja log v7\n"); + fprintf(f, "456\t789\t789\tout2\t%" PRIx64 "\n", + BuildLog::LogEntry::HashCommand("command2")); fclose(f); - string err; + std::string err; BuildLog log; EXPECT_TRUE(log.Load(kTestFilename, &err)); ASSERT_EQ("", err); @@ -217,19 +226,69 @@ TEST_F(BuildLogTest, DuplicateVersionHeader) { ASSERT_NO_FATAL_FAILURE(AssertHash("command2", e->command_hash)); } +struct TestDiskInterface : public DiskInterface { + virtual TimeStamp Stat(const std::string& path, std::string* err) const { + return 4; + } + virtual bool WriteFile(const std::string& path, const std::string& contents) { + assert(false); + return true; + } + virtual bool MakeDir(const std::string& path) { + assert(false); + return false; + } + virtual Status ReadFile(const std::string& path, std::string* contents, + std::string* err) { + assert(false); + return NotFound; + } + virtual int RemoveFile(const std::string& path) { + assert(false); + return 0; + } +}; + +TEST_F(BuildLogTest, Restat) { + FILE* f = fopen(kTestFilename, "wb"); + fprintf(f, "# ninja log v7\n" + "1\t2\t3\tout\tcommand\n"); + fclose(f); + std::string err; + BuildLog log; + EXPECT_TRUE(log.Load(kTestFilename, &err)); + ASSERT_EQ("", err); + BuildLog::LogEntry* e = log.LookupByOutput("out"); + ASSERT_EQ(3, e->mtime); + + TestDiskInterface testDiskInterface; + char out2[] = { 'o', 'u', 't', '2', 0 }; + char* filter2[] = { out2 }; + EXPECT_TRUE(log.Restat(kTestFilename, testDiskInterface, 1, filter2, &err)); + ASSERT_EQ("", err); + e = log.LookupByOutput("out"); + ASSERT_EQ(3, e->mtime); // unchanged, since the filter doesn't match + + EXPECT_TRUE(log.Restat(kTestFilename, testDiskInterface, 0, NULL, &err)); + ASSERT_EQ("", err); + e = log.LookupByOutput("out"); + ASSERT_EQ(4, e->mtime); +} + TEST_F(BuildLogTest, VeryLongInputLine) { // Ninja's build log buffer is currently 256kB. Lines longer than that are // silently ignored, but don't affect parsing of other lines. FILE* f = fopen(kTestFilename, "wb"); - fprintf(f, "# ninja log v4\n"); + fprintf(f, "# ninja log v7\n"); fprintf(f, "123\t456\t456\tout\tcommand start"); for (size_t i = 0; i < (512 << 10) / strlen(" more_command"); ++i) fputs(" more_command", f); fprintf(f, "\n"); - fprintf(f, "456\t789\t789\tout2\tcommand2\n"); + fprintf(f, "456\t789\t789\tout2\t%" PRIx64 "\n", + BuildLog::LogEntry::HashCommand("command2")); fclose(f); - string err; + std::string err; BuildLog log; EXPECT_TRUE(log.Load(kTestFilename, &err)); ASSERT_EQ("", err); @@ -275,7 +334,7 @@ TEST_F(BuildLogRecompactTest, Recompact) { "build out2: cat in\n"); BuildLog log1; - string err; + std::string err; EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err)); ASSERT_EQ("", err); // Record the same edge several times, to trigger recompaction diff --git a/src/build_test.cc b/src/build_test.cc index b5dbc6c1f6..856b0365ef 100644 --- a/src/build_test.cc +++ b/src/build_test.cc @@ -15,12 +15,18 @@ #include "build.h" #include +#include +#include #include "build_log.h" #include "deps_log.h" +#include "exit_status.h" #include "graph.h" +#include "status_printer.h" #include "test.h" +using namespace std; + struct CompareEdgesByOutput { static bool cmp(const Edge* a, const Edge* b) { return a->outputs_[0]->path() < b->outputs_[0]->path(); @@ -47,6 +53,14 @@ struct PlanTest : public StateTestWithBuiltinRules { sort(ret->begin(), ret->end(), CompareEdgesByOutput::cmp); } + void PrepareForTarget(const char* node, BuildLog *log=NULL) { + string err; + EXPECT_TRUE(plan_.AddTarget(GetNode(node), &err)); + ASSERT_EQ("", err); + plan_.PrepareQueue(); + ASSERT_TRUE(plan_.more_to_do()); + } + void TestPoolWithDepthOne(const char *test_case); }; @@ -56,10 +70,7 @@ TEST_F(PlanTest, Basic) { "build mid: cat in\n")); GetNode("mid")->MarkDirty(); GetNode("out")->MarkDirty(); - string err; - EXPECT_TRUE(plan_.AddTarget(GetNode("out"), &err)); - ASSERT_EQ("", err); - ASSERT_TRUE(plan_.more_to_do()); + PrepareForTarget("out"); Edge* edge = plan_.FindWork(); ASSERT_TRUE(edge); @@ -68,6 +79,7 @@ TEST_F(PlanTest, Basic) { ASSERT_FALSE(plan_.FindWork()); + string err; plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); ASSERT_EQ("", err); @@ -92,15 +104,12 @@ TEST_F(PlanTest, DoubleOutputDirect) { GetNode("mid1")->MarkDirty(); GetNode("mid2")->MarkDirty(); GetNode("out")->MarkDirty(); - - string err; - EXPECT_TRUE(plan_.AddTarget(GetNode("out"), &err)); - ASSERT_EQ("", err); - ASSERT_TRUE(plan_.more_to_do()); + PrepareForTarget("out"); Edge* edge; edge = plan_.FindWork(); ASSERT_TRUE(edge); // cat in + string err; plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); ASSERT_EQ("", err); @@ -125,14 +134,12 @@ TEST_F(PlanTest, DoubleOutputIndirect) { GetNode("b1")->MarkDirty(); GetNode("b2")->MarkDirty(); GetNode("out")->MarkDirty(); - string err; - EXPECT_TRUE(plan_.AddTarget(GetNode("out"), &err)); - ASSERT_EQ("", err); - ASSERT_TRUE(plan_.more_to_do()); + PrepareForTarget("out"); Edge* edge; edge = plan_.FindWork(); ASSERT_TRUE(edge); // cat in + string err; plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); ASSERT_EQ("", err); @@ -166,15 +173,12 @@ TEST_F(PlanTest, DoubleDependent) { GetNode("a1")->MarkDirty(); GetNode("a2")->MarkDirty(); GetNode("out")->MarkDirty(); - - string err; - EXPECT_TRUE(plan_.AddTarget(GetNode("out"), &err)); - ASSERT_EQ("", err); - ASSERT_TRUE(plan_.more_to_do()); + PrepareForTarget("out"); Edge* edge; edge = plan_.FindWork(); ASSERT_TRUE(edge); // cat in + string err; plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); ASSERT_EQ("", err); @@ -206,6 +210,7 @@ void PlanTest::TestPoolWithDepthOne(const char* test_case) { ASSERT_EQ("", err); EXPECT_TRUE(plan_.AddTarget(GetNode("out2"), &err)); ASSERT_EQ("", err); + plan_.PrepareQueue(); ASSERT_TRUE(plan_.more_to_do()); Edge* edge = plan_.FindWork(); @@ -281,10 +286,7 @@ TEST_F(PlanTest, PoolsWithDepthTwo) { GetNode("outb" + string(1, '1' + static_cast(i)))->MarkDirty(); } GetNode("allTheThings")->MarkDirty(); - - string err; - EXPECT_TRUE(plan_.AddTarget(GetNode("allTheThings"), &err)); - ASSERT_EQ("", err); + PrepareForTarget("allTheThings"); deque edges; FindWorkSorted(&edges, 5); @@ -303,6 +305,7 @@ TEST_F(PlanTest, PoolsWithDepthTwo) { ASSERT_EQ("outb3", edge->outputs_[0]->path()); // finish out1 + string err; plan_.EdgeFinished(edges.front(), Plan::kEdgeSucceeded, &err); ASSERT_EQ("", err); edges.pop_front(); @@ -360,10 +363,7 @@ TEST_F(PlanTest, PoolWithRedundantEdges) { GetNode("bar.cpp.obj")->MarkDirty(); GetNode("libfoo.a")->MarkDirty(); GetNode("all")->MarkDirty(); - string err; - EXPECT_TRUE(plan_.AddTarget(GetNode("all"), &err)); - ASSERT_EQ("", err); - ASSERT_TRUE(plan_.more_to_do()); + PrepareForTarget("all"); Edge* edge = NULL; @@ -372,6 +372,7 @@ TEST_F(PlanTest, PoolWithRedundantEdges) { edge = initial_edges[1]; // Foo first ASSERT_EQ("foo.cpp", edge->outputs_[0]->path()); + string err; plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err); ASSERT_EQ("", err); @@ -436,6 +437,7 @@ TEST_F(PlanTest, PoolWithFailingEdge) { ASSERT_EQ("", err); EXPECT_TRUE(plan_.AddTarget(GetNode("out2"), &err)); ASSERT_EQ("", err); + plan_.PrepareQueue(); ASSERT_TRUE(plan_.more_to_do()); Edge* edge = plan_.FindWork(); @@ -464,13 +466,63 @@ TEST_F(PlanTest, PoolWithFailingEdge) { ASSERT_EQ(0, edge); } +TEST_F(PlanTest, PriorityWithoutBuildLog) { + // Without a build log, the critical time is equivalent to graph + // depth. Test with the following graph: + // a2 + // | + // a1 b1 + // | | | + // a0 b0 c0 + // \ | / + // out + + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "rule r\n" + " command = unused\n" + "build out: r a0 b0 c0\n" + "build a0: r a1\n" + "build a1: r a2\n" + "build b0: r b1\n" + "build c0: r b1\n" + )); + GetNode("a1")->MarkDirty(); + GetNode("a0")->MarkDirty(); + GetNode("b0")->MarkDirty(); + GetNode("c0")->MarkDirty(); + GetNode("out")->MarkDirty(); + BuildLog log; + PrepareForTarget("out", &log); + + EXPECT_EQ(GetNode("out")->in_edge()->critical_path_weight(), 1); + EXPECT_EQ(GetNode("a0")->in_edge()->critical_path_weight(), 2); + EXPECT_EQ(GetNode("b0")->in_edge()->critical_path_weight(), 2); + EXPECT_EQ(GetNode("c0")->in_edge()->critical_path_weight(), 2); + EXPECT_EQ(GetNode("a1")->in_edge()->critical_path_weight(), 3); + + const int n_edges = 5; + const char *expected_order[n_edges] = { + "a1", "a0", "b0", "c0", "out"}; + for (int i = 0; i < n_edges; ++i) { + Edge* edge = plan_.FindWork(); + ASSERT_TRUE(edge != nullptr); + EXPECT_EQ(expected_order[i], edge->outputs_[0]->path()); + + std::string err; + ASSERT_TRUE(plan_.EdgeFinished(edge, Plan::kEdgeSucceeded, &err)); + EXPECT_EQ(err, ""); + } + + EXPECT_FALSE(plan_.FindWork()); +} + /// Fake implementation of CommandRunner, useful for tests. struct FakeCommandRunner : public CommandRunner { explicit FakeCommandRunner(VirtualFileSystem* fs) : max_active_edges_(1), fs_(fs) {} // CommandRunner impl - virtual bool CanRunMore(); + virtual size_t CanRunMore() const; virtual bool StartCommand(Edge* edge); virtual bool WaitForCommand(Result* result); virtual vector GetActiveEdges(); @@ -483,11 +535,14 @@ struct FakeCommandRunner : public CommandRunner { }; struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser { - BuildTest() : config_(MakeConfig()), command_runner_(&fs_), - builder_(&state_, config_, NULL, NULL, &fs_), - status_(config_) { + BuildTest() : config_(MakeConfig()), command_runner_(&fs_), status_(config_), + builder_(&state_, config_, NULL, NULL, &fs_, &status_, 0) { } + explicit BuildTest(DepsLog* log) + : config_(MakeConfig()), command_runner_(&fs_), status_(config_), + builder_(&state_, config_, NULL, log, &fs_, &status_, 0) {} + virtual void SetUp() { StateTestWithBuiltinRules::SetUp(); @@ -526,9 +581,8 @@ struct BuildTest : public StateTestWithBuiltinRules, public BuildLogUser { BuildConfig config_; FakeCommandRunner command_runner_; VirtualFileSystem fs_; + StatusPrinter status_; Builder builder_; - - BuildStatus status_; }; void BuildTest::RebuildTarget(const string& target, const char* manifest, @@ -557,20 +611,23 @@ void BuildTest::RebuildTarget(const string& target, const char* manifest, pdeps_log = &deps_log; } - Builder builder(pstate, config_, pbuild_log, pdeps_log, &fs_); + Builder builder(pstate, config_, pbuild_log, pdeps_log, &fs_, &status_, 0); EXPECT_TRUE(builder.AddTarget(target, &err)); command_runner_.commands_ran_.clear(); builder.command_runner_.reset(&command_runner_); if (!builder.AlreadyUpToDate()) { - bool build_res = builder.Build(&err); - EXPECT_TRUE(build_res); + ExitStatus build_res = builder.Build(&err); + EXPECT_EQ(build_res, ExitSuccess); } builder.command_runner_.release(); } -bool FakeCommandRunner::CanRunMore() { - return active_edges_.size() < max_active_edges_; +size_t FakeCommandRunner::CanRunMore() const { + if (active_edges_.size() < max_active_edges_) + return SIZE_MAX; + + return 0; } bool FakeCommandRunner::StartCommand(Edge* edge) { @@ -582,6 +639,8 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { edge->rule().name() == "cat_rsp" || edge->rule().name() == "cat_rsp_out" || edge->rule().name() == "cc" || + edge->rule().name() == "cp_multi_msvc" || + edge->rule().name() == "cp_multi_gcc" || edge->rule().name() == "touch" || edge->rule().name() == "touch-interrupt" || edge->rule().name() == "touch-fail-tick2") { @@ -602,6 +661,52 @@ bool FakeCommandRunner::StartCommand(Edge* edge) { if (fs_->ReadFile(edge->inputs_[0]->path(), &content, &err) == DiskInterface::Okay) fs_->WriteFile(edge->outputs_[0]->path(), content); + } else if (edge->rule().name() == "touch-implicit-dep-out") { + string dep = edge->GetBinding("test_dependency"); + fs_->Tick(); + fs_->Create(dep, ""); + fs_->Tick(); + for (vector::iterator out = edge->outputs_.begin(); + out != edge->outputs_.end(); ++out) { + fs_->Create((*out)->path(), ""); + } + } else if (edge->rule().name() == "touch-out-implicit-dep") { + string dep = edge->GetBinding("test_dependency"); + for (vector::iterator out = edge->outputs_.begin(); + out != edge->outputs_.end(); ++out) { + fs_->Create((*out)->path(), ""); + } + fs_->Tick(); + fs_->Create(dep, ""); + } else if (edge->rule().name() == "generate-depfile") { + string dep = edge->GetBinding("test_dependency"); + bool touch_dep = edge->GetBindingBool("touch_dependency"); + string depfile = edge->GetUnescapedDepfile(); + if (touch_dep) { + fs_->Tick(); + fs_->Create(dep, ""); + } + string contents; + for (vector::iterator out = edge->outputs_.begin(); + out != edge->outputs_.end(); ++out) { + contents += (*out)->path() + ": " + dep + "\n"; + fs_->Create((*out)->path(), ""); + } + fs_->Create(depfile, contents); + } else if (edge->rule().name() == "long-cc") { + string dep = edge->GetBinding("test_dependency"); + string depfile = edge->GetUnescapedDepfile(); + string contents; + for (vector::iterator out = edge->outputs_.begin(); + out != edge->outputs_.end(); ++out) { + fs_->Tick(); + fs_->Tick(); + fs_->Tick(); + fs_->Create((*out)->path(), ""); + contents += (*out)->path() + ": " + dep + "\n"; + } + if (!dep.empty() && !depfile.empty()) + fs_->Create(depfile, contents); } else { printf("unknown command\n"); return false; @@ -643,12 +748,32 @@ bool FakeCommandRunner::WaitForCommand(Result* result) { return true; } + if (edge->rule().name() == "cp_multi_msvc") { + const std::string prefix = edge->GetBinding("msvc_deps_prefix"); + for (std::vector::iterator in = edge->inputs_.begin(); + in != edge->inputs_.end(); ++in) { + result->output += prefix + (*in)->path() + '\n'; + } + } + if (edge->rule().name() == "fail" || (edge->rule().name() == "touch-fail-tick2" && fs_->now_ == 2)) result->status = ExitFailure; else result->status = ExitSuccess; + // This rule simulates an external process modifying files while the build command runs. + // See TestInputMtimeRaceCondition and TestInputMtimeRaceConditionWithDepFile. + // Note: only the first and third time the rule is run per test is the file modified, so + // the test can verify that subsequent runs without the race have no work to do. + if (edge->rule().name() == "long-cc") { + string dep = edge->GetBinding("test_dependency"); + if (fs_->now_ == 4) + fs_->files_[dep].mtime = 3; + if (fs_->now_ == 10) + fs_->files_[dep].mtime = 9; + } + // Provide a way for test cases to verify when an edge finishes that // some other edge is still active. This is useful for test cases // covering behavior involving multiple active edges. @@ -657,7 +782,7 @@ bool FakeCommandRunner::WaitForCommand(Result* result) { bool verify_active_edge_found = false; for (vector::iterator i = active_edges_.begin(); i != active_edges_.end(); ++i) { - if ((*i)->outputs_.size() >= 1 && + if (!(*i)->outputs_.empty() && (*i)->outputs_[0]->path() == verify_active_edge) { verify_active_edge_found = true; } @@ -699,7 +824,7 @@ TEST_F(BuildTest, OneStep) { string err; EXPECT_TRUE(builder_.AddTarget("cat1", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); @@ -713,7 +838,7 @@ TEST_F(BuildTest, OneStep2) { string err; EXPECT_TRUE(builder_.AddTarget("cat1", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); @@ -724,7 +849,7 @@ TEST_F(BuildTest, TwoStep) { string err; EXPECT_TRUE(builder_.AddTarget("cat12", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); // Depending on how the pointers work out, we could've ran @@ -744,7 +869,7 @@ TEST_F(BuildTest, TwoStep) { state_.Reset(); EXPECT_TRUE(builder_.AddTarget("cat12", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(5u, command_runner_.commands_ran_.size()); EXPECT_EQ("cat in1 in2 > cat2", command_runner_.commands_ran_[3]); @@ -762,7 +887,7 @@ TEST_F(BuildTest, TwoOutputs) { string err; EXPECT_TRUE(builder_.AddTarget("out1", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); EXPECT_EQ("touch out1 out2", command_runner_.commands_ran_[0]); @@ -778,7 +903,7 @@ TEST_F(BuildTest, ImplicitOutput) { string err; EXPECT_TRUE(builder_.AddTarget("out.imp", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[0]); @@ -800,7 +925,7 @@ TEST_F(BuildTest, MultiOutIn) { string err; EXPECT_TRUE(builder_.AddTarget("out", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); } @@ -816,7 +941,7 @@ TEST_F(BuildTest, Chain) { string err; EXPECT_TRUE(builder_.AddTarget("c5", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(4u, command_runner_.commands_ran_.size()); @@ -836,7 +961,7 @@ TEST_F(BuildTest, Chain) { EXPECT_TRUE(builder_.AddTarget("c5", &err)); ASSERT_EQ("", err); EXPECT_FALSE(builder_.AlreadyUpToDate()); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ(2u, command_runner_.commands_ran_.size()); // 3->4, 4->5 } @@ -856,6 +981,14 @@ TEST_F(BuildTest, MissingTarget) { EXPECT_EQ("unknown target: 'meow'", err); } +TEST_F(BuildTest, MissingInputTarget) { + // Target is a missing input file + string err; + Dirty("in1"); + EXPECT_FALSE(builder_.AddTarget("in1", &err)); + EXPECT_EQ("'in1' missing and no known rule to make it", err); +} + TEST_F(BuildTest, MakeDirs) { string err; @@ -869,7 +1002,7 @@ TEST_F(BuildTest, MakeDirs) { EXPECT_TRUE(builder_.AddTarget("subdir/dir2/file", &err)); EXPECT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(2u, fs_.directories_made_.size()); EXPECT_EQ("subdir", fs_.directories_made_[0]); @@ -905,9 +1038,19 @@ TEST_F(BuildTest, DepFileOK) { ASSERT_EQ(1u, fs_.files_read_.size()); EXPECT_EQ("foo.o.d", fs_.files_read_[0]); - // Expect three new edges: one generating foo.o, and two more from - // loading the depfile. - ASSERT_EQ(orig_edges + 3, (int)state_.edges_.size()); + // Expect one new edge generating foo.o. Loading the depfile should have + // added nodes, but not phony edges to the graph. + ASSERT_EQ(orig_edges + 1, (int)state_.edges_.size()); + + // Verify that nodes for blah.h and bar.h were added and that they + // are marked as generated by a dep loader. + ASSERT_FALSE(state_.LookupNode("foo.o")->generated_by_dep_loader()); + ASSERT_FALSE(state_.LookupNode("foo.c")->generated_by_dep_loader()); + ASSERT_TRUE(state_.LookupNode("blah.h")); + ASSERT_TRUE(state_.LookupNode("blah.h")->generated_by_dep_loader()); + ASSERT_TRUE(state_.LookupNode("bar.h")); + ASSERT_TRUE(state_.LookupNode("bar.h")->generated_by_dep_loader()); + // Expect our edge to now have three inputs: foo.c and two headers. ASSERT_EQ(3u, edge->inputs_.size()); @@ -944,7 +1087,7 @@ TEST_F(BuildTest, EncounterReadyTwice) { EXPECT_TRUE(builder_.AddTarget("a", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(2u, command_runner_.commands_ran_.size()); } @@ -977,7 +1120,7 @@ TEST_F(BuildTest, OrderOnlyDeps) { ASSERT_EQ("cc foo.c", edge->EvaluateCommand()); // explicit dep dirty, expect a rebuild. - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); @@ -992,7 +1135,7 @@ TEST_F(BuildTest, OrderOnlyDeps) { command_runner_.commands_ran_.clear(); state_.Reset(); EXPECT_TRUE(builder_.AddTarget("foo.o", &err)); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); @@ -1014,7 +1157,7 @@ TEST_F(BuildTest, OrderOnlyDeps) { command_runner_.commands_ran_.clear(); state_.Reset(); EXPECT_TRUE(builder_.AddTarget("foo.o", &err)); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); } @@ -1032,7 +1175,7 @@ TEST_F(BuildTest, RebuildOrderOnlyDeps) { // foo.o and order-only dep dirty, build both. EXPECT_TRUE(builder_.AddTarget("foo.o", &err)); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(2u, command_runner_.commands_ran_.size()); @@ -1048,7 +1191,7 @@ TEST_F(BuildTest, RebuildOrderOnlyDeps) { command_runner_.commands_ran_.clear(); state_.Reset(); EXPECT_TRUE(builder_.AddTarget("foo.o", &err)); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); ASSERT_EQ("cc oo.h.in", command_runner_.commands_ran_[0]); @@ -1060,7 +1203,7 @@ TEST_F(BuildTest, RebuildOrderOnlyDeps) { command_runner_.commands_ran_.clear(); state_.Reset(); EXPECT_TRUE(builder_.AddTarget("foo.o", &err)); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); ASSERT_EQ("cc oo.h.in", command_runner_.commands_ran_[0]); @@ -1073,7 +1216,6 @@ TEST_F(BuildTest, DepFileCanonicalize) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule cc\n command = cc $in\n depfile = $out.d\n" "build gen/stuff\\things/foo.o: cc x\\y/z\\foo.c\n")); - Edge* edge = state_.edges_.back(); fs_.Create("x/y/z/foo.c", ""); GetNode("bar.h")->MarkDirty(); // Mark bar.h as missing. @@ -1086,10 +1228,10 @@ TEST_F(BuildTest, DepFileCanonicalize) { // The depfile path does not get Canonicalize as it seems unnecessary. EXPECT_EQ("gen/stuff\\things/foo.o.d", fs_.files_read_[0]); - // Expect three new edges: one generating foo.o, and two more from - // loading the depfile. - ASSERT_EQ(orig_edges + 3, (int)state_.edges_.size()); + // Expect one new edge enerating foo.o. + ASSERT_EQ(orig_edges + 1, (int)state_.edges_.size()); // Expect our edge to now have three inputs: foo.c and two headers. + Edge* edge = state_.edges_.back(); ASSERT_EQ(3u, edge->inputs_.size()); // Expect the command line we generate to only use the original input, and @@ -1110,7 +1252,7 @@ TEST_F(BuildTest, Phony) { // Only one command to run, because phony runs no command. EXPECT_FALSE(builder_.AlreadyUpToDate()); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); } @@ -1141,6 +1283,153 @@ TEST_F(BuildTest, PhonySelfReference) { EXPECT_TRUE(builder_.AlreadyUpToDate()); } +// There are 6 different cases for phony rules: +// +// 1. output edge does not exist, inputs are not real +// 2. output edge does not exist, no inputs +// 3. output edge does not exist, inputs are real, newest mtime is M +// 4. output edge is real, inputs are not real +// 5. output edge is real, no inputs +// 6. output edge is real, inputs are real, newest mtime is M +// +// Expected results : +// 1. Edge is marked as clean, mtime is newest mtime of dependents. +// Touching inputs will cause dependents to rebuild. +// 2. Edge is marked as dirty, causing dependent edges to always rebuild +// 3. Edge is marked as clean, mtime is newest mtime of dependents. +// Touching inputs will cause dependents to rebuild. +// 4. Edge is marked as clean, mtime is newest mtime of dependents. +// Touching inputs will cause dependents to rebuild. +// 5. Edge is marked as dirty, causing dependent edges to always rebuild +// 6. Edge is marked as clean, mtime is newest mtime of dependents. +// Touching inputs will cause dependents to rebuild. +void TestPhonyUseCase(BuildTest* t, int i) { + State& state_ = t->state_; + Builder& builder_ = t->builder_; + FakeCommandRunner& command_runner_ = t->command_runner_; + VirtualFileSystem& fs_ = t->fs_; + + string err; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"build notreal: phony blank\n" +"build phony1: phony notreal\n" +"build phony2: phony\n" +"build phony3: phony blank\n" +"build phony4: phony notreal\n" +"build phony5: phony\n" +"build phony6: phony blank\n" +"\n" +"build test1: touch phony1\n" +"build test2: touch phony2\n" +"build test3: touch phony3\n" +"build test4: touch phony4\n" +"build test5: touch phony5\n" +"build test6: touch phony6\n" +)); + + // Set up test. + builder_.command_runner_.release(); // BuildTest owns the CommandRunner + builder_.command_runner_.reset(&command_runner_); + + fs_.Create("blank", ""); // a "real" file + EXPECT_TRUE(builder_.AddTarget("test1", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test2", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test3", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test4", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test5", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder_.AddTarget("test6", &err)); + ASSERT_EQ("", err); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + ASSERT_EQ("", err); + + string ci; + ci += static_cast('0' + i); + + // Tests 1, 3, 4, and 6 should rebuild when the input is updated. + if (i != 2 && i != 5) { + Node* testNode = t->GetNode("test" + ci); + Node* phonyNode = t->GetNode("phony" + ci); + Node* inputNode = t->GetNode("blank"); + + state_.Reset(); + TimeStamp startTime = fs_.now_; + + // Build number 1 + EXPECT_TRUE(builder_.AddTarget("test" + ci, &err)); + ASSERT_EQ("", err); + if (!builder_.AlreadyUpToDate()) { + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + } + ASSERT_EQ("", err); + + // Touch the input file + state_.Reset(); + command_runner_.commands_ran_.clear(); + fs_.Tick(); + fs_.Create("blank", ""); // a "real" file + EXPECT_TRUE(builder_.AddTarget("test" + ci, &err)); + ASSERT_EQ("", err); + + // Second build, expect testN edge to be rebuilt + // and phonyN node's mtime to be updated. + EXPECT_FALSE(builder_.AlreadyUpToDate()); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + ASSERT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ(string("touch test") + ci, command_runner_.commands_ran_[0]); + EXPECT_TRUE(builder_.AlreadyUpToDate()); + + TimeStamp inputTime = inputNode->mtime(); + + EXPECT_FALSE(phonyNode->exists()); + EXPECT_FALSE(phonyNode->dirty()); + + EXPECT_GT(phonyNode->mtime(), startTime); + EXPECT_EQ(phonyNode->mtime(), inputTime); + ASSERT_TRUE(testNode->Stat(&fs_, &err)); + EXPECT_TRUE(testNode->exists()); + EXPECT_GT(testNode->mtime(), startTime); + } else { + // Tests 2 and 5: Expect dependents to always rebuild. + + state_.Reset(); + command_runner_.commands_ran_.clear(); + fs_.Tick(); + command_runner_.commands_ran_.clear(); + EXPECT_TRUE(builder_.AddTarget("test" + ci, &err)); + ASSERT_EQ("", err); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + ASSERT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("touch test" + ci, command_runner_.commands_ran_[0]); + + state_.Reset(); + command_runner_.commands_ran_.clear(); + EXPECT_TRUE(builder_.AddTarget("test" + ci, &err)); + ASSERT_EQ("", err); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + ASSERT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("touch test" + ci, command_runner_.commands_ran_[0]); + } +} + +TEST_F(BuildTest, PhonyUseCase1) { TestPhonyUseCase(this, 1); } +TEST_F(BuildTest, PhonyUseCase2) { TestPhonyUseCase(this, 2); } +TEST_F(BuildTest, PhonyUseCase3) { TestPhonyUseCase(this, 3); } +TEST_F(BuildTest, PhonyUseCase4) { TestPhonyUseCase(this, 4); } +TEST_F(BuildTest, PhonyUseCase5) { TestPhonyUseCase(this, 5); } +TEST_F(BuildTest, PhonyUseCase6) { TestPhonyUseCase(this, 6); } + TEST_F(BuildTest, Fail) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule fail\n" @@ -1151,7 +1440,7 @@ TEST_F(BuildTest, Fail) { EXPECT_TRUE(builder_.AddTarget("out1", &err)); ASSERT_EQ("", err); - EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitFailure); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); ASSERT_EQ("subcommand failed", err); } @@ -1172,7 +1461,7 @@ TEST_F(BuildTest, SwallowFailures) { EXPECT_TRUE(builder_.AddTarget("all", &err)); ASSERT_EQ("", err); - EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitFailure); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); ASSERT_EQ("subcommands failed", err); } @@ -1193,7 +1482,7 @@ TEST_F(BuildTest, SwallowFailuresLimit) { EXPECT_TRUE(builder_.AddTarget("final", &err)); ASSERT_EQ("", err); - EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitFailure); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); ASSERT_EQ("cannot make progress due to previous errors", err); } @@ -1217,7 +1506,7 @@ TEST_F(BuildTest, SwallowFailuresPool) { EXPECT_TRUE(builder_.AddTarget("final", &err)); ASSERT_EQ("", err); - EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitFailure); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); ASSERT_EQ("cannot make progress due to previous errors", err); } @@ -1255,6 +1544,78 @@ struct BuildWithLogTest : public BuildTest { BuildLog build_log_; }; +TEST_F(BuildWithLogTest, ImplicitGeneratedOutOfDate) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +" generator = 1\n" +"build out.imp: touch | in\n")); + fs_.Create("out.imp", ""); + fs_.Tick(); + fs_.Create("in", ""); + + string err; + + EXPECT_TRUE(builder_.AddTarget("out.imp", &err)); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + + EXPECT_TRUE(GetNode("out.imp")->dirty()); +} + +TEST_F(BuildWithLogTest, ImplicitGeneratedOutOfDate2) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch-implicit-dep-out\n" +" command = sleep 1 ; touch $test_dependency ; sleep 1 ; touch $out\n" +" generator = 1\n" +"build out.imp: touch-implicit-dep-out | inimp inimp2\n" +" test_dependency = inimp\n")); + fs_.Create("inimp", ""); + fs_.Create("out.imp", ""); + fs_.Tick(); + fs_.Create("inimp2", ""); + fs_.Tick(); + + string err; + + EXPECT_TRUE(builder_.AddTarget("out.imp", &err)); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_TRUE(builder_.AlreadyUpToDate()); + + command_runner_.commands_ran_.clear(); + state_.Reset(); + builder_.Cleanup(); + builder_.plan_.Reset(); + + EXPECT_TRUE(builder_.AddTarget("out.imp", &err)); + EXPECT_TRUE(builder_.AlreadyUpToDate()); + EXPECT_FALSE(GetNode("out.imp")->dirty()); + + command_runner_.commands_ran_.clear(); + state_.Reset(); + builder_.Cleanup(); + builder_.plan_.Reset(); + + fs_.Tick(); + fs_.Create("inimp", ""); + + EXPECT_TRUE(builder_.AddTarget("out.imp", &err)); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_TRUE(builder_.AlreadyUpToDate()); + + command_runner_.commands_ran_.clear(); + state_.Reset(); + builder_.Cleanup(); + builder_.plan_.Reset(); + + EXPECT_TRUE(builder_.AddTarget("out.imp", &err)); + EXPECT_TRUE(builder_.AlreadyUpToDate()); + EXPECT_FALSE(GetNode("out.imp")->dirty()); +} + TEST_F(BuildWithLogTest, NotInLogButOnDisk) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "rule cc\n" @@ -1276,7 +1637,7 @@ TEST_F(BuildWithLogTest, NotInLogButOnDisk) { state_.Reset(); EXPECT_TRUE(builder_.AddTarget("out1", &err)); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_TRUE(builder_.AlreadyUpToDate()); } @@ -1292,7 +1653,7 @@ TEST_F(BuildWithLogTest, RebuildAfterFailure) { // Run once successfully to get out1 in the log EXPECT_TRUE(builder_.AddTarget("out1", &err)); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); EXPECT_EQ(1u, command_runner_.commands_ran_.size()); @@ -1306,7 +1667,7 @@ TEST_F(BuildWithLogTest, RebuildAfterFailure) { // Run again with a failure that updates the output file timestamp EXPECT_TRUE(builder_.AddTarget("out1", &err)); - EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitFailure); EXPECT_EQ("subcommand failed", err); EXPECT_EQ(1u, command_runner_.commands_ran_.size()); @@ -1320,7 +1681,7 @@ TEST_F(BuildWithLogTest, RebuildAfterFailure) { // Run again, should rerun even though the output file is up to date on disk EXPECT_TRUE(builder_.AddTarget("out1", &err)); EXPECT_FALSE(builder_.AlreadyUpToDate()); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ(1u, command_runner_.commands_ran_.size()); EXPECT_EQ("", err); } @@ -1338,7 +1699,7 @@ TEST_F(BuildWithLogTest, RebuildWithNoInputs) { EXPECT_TRUE(builder_.AddTarget("out1", &err)); EXPECT_TRUE(builder_.AddTarget("out2", &err)); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); EXPECT_EQ(2u, command_runner_.commands_ran_.size()); @@ -1351,7 +1712,7 @@ TEST_F(BuildWithLogTest, RebuildWithNoInputs) { EXPECT_TRUE(builder_.AddTarget("out1", &err)); EXPECT_TRUE(builder_.AddTarget("out2", &err)); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); EXPECT_EQ(1u, command_runner_.commands_ran_.size()); } @@ -1382,10 +1743,10 @@ TEST_F(BuildWithLogTest, RestatTest) { string err; EXPECT_TRUE(builder_.AddTarget("out3", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); - EXPECT_EQ("[3/3]", builder_.status_->FormatProgressStatus("[%s/%t]", - BuildStatus::kEdgeStarted)); + EXPECT_EQ(size_t(3), command_runner_.commands_ran_.size()); + EXPECT_EQ(3, builder_.plan_.command_edge_count()); command_runner_.commands_ran_.clear(); state_.Reset(); @@ -1396,7 +1757,7 @@ TEST_F(BuildWithLogTest, RestatTest) { // touch out2, we should cancel the build of out3. EXPECT_TRUE(builder_.AddTarget("out3", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ(2u, command_runner_.commands_ran_.size()); // If we run again, it should be a no-op, because the build log has recorded @@ -1417,7 +1778,7 @@ TEST_F(BuildWithLogTest, RestatTest) { state_.Reset(); EXPECT_TRUE(builder_.AddTarget("out3", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ(2u, command_runner_.commands_ran_.size()); } @@ -1444,7 +1805,7 @@ TEST_F(BuildWithLogTest, RestatMissingFile) { string err; EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); command_runner_.commands_ran_.clear(); state_.Reset(); @@ -1458,7 +1819,7 @@ TEST_F(BuildWithLogTest, RestatMissingFile) { // we shouldn't run the dependent build. EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); } @@ -1480,7 +1841,7 @@ TEST_F(BuildWithLogTest, RestatSingleDependentOutputDirty) { string err; EXPECT_TRUE(builder_.AddTarget("out4", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); @@ -1492,12 +1853,12 @@ TEST_F(BuildWithLogTest, RestatSingleDependentOutputDirty) { // out2 and out3 will be built even though "in" is not touched when built. // Then, since out2 is rebuilt, out4 should be rebuilt -- the restat on the // "true" rule should not lead to the "touch" edge writing out2 and out3 being - // cleard. + // cleared. command_runner_.commands_ran_.clear(); state_.Reset(); EXPECT_TRUE(builder_.AddTarget("out4", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); } @@ -1529,7 +1890,7 @@ TEST_F(BuildWithLogTest, RestatMissingInput) { string err; EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ(2u, command_runner_.commands_ran_.size()); // See that an entry in the logfile is created, capturing @@ -1547,7 +1908,7 @@ TEST_F(BuildWithLogTest, RestatMissingInput) { state_.Reset(); EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); // Check that the logfile entry remains correctly set @@ -1556,6 +1917,79 @@ TEST_F(BuildWithLogTest, RestatMissingInput) { ASSERT_EQ(restat_mtime, log_entry->mtime); } +TEST_F(BuildWithLogTest, RestatInputChangesDueToRule) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule generate-depfile\n" +" command = sleep 1 ; touch $touch_dependency; touch $out ; echo \"$out: $test_dependency\" > $depfile\n" +"build out1: generate-depfile || cat1\n" +" test_dependency = in2\n" +" touch_dependency = 1\n" +" restat = 1\n" +" depfile = out.d\n")); + + // Perform the first build. out1 is a restat rule, so its recorded mtime in the build + // log should be the time the command completes, not the time the command started. One + // of out1's discovered dependencies will have a newer mtime than when out1 started + // running, due to its command touching the dependency itself. + string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + ASSERT_EQ("", err); + EXPECT_EQ(size_t(2), command_runner_.commands_ran_.size()); + EXPECT_EQ(2, builder_.plan_.command_edge_count()); + BuildLog::LogEntry* log_entry = build_log_.LookupByOutput("out1"); + ASSERT_TRUE(NULL != log_entry); + ASSERT_EQ(2u, log_entry->mtime); + + command_runner_.commands_ran_.clear(); + state_.Reset(); + builder_.Cleanup(); + builder_.plan_.Reset(); + + fs_.Tick(); + fs_.Create("in1", ""); + + // Touching a dependency of an order-only dependency of out1 should not cause out1 to + // rebuild. If out1 were not a restat rule, then it would rebuild here because its + // recorded mtime would have been an earlier mtime than its most recent input's (in2) + // mtime + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(!state_.GetNode("out1", 0)->dirty()); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + ASSERT_EQ("", err); + EXPECT_EQ(size_t(1), command_runner_.commands_ran_.size()); + EXPECT_EQ(1, builder_.plan_.command_edge_count()); +} + +TEST_F(BuildWithLogTest, GeneratedPlainDepfileMtime) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule generate-depfile\n" +" command = touch $out ; echo \"$out: $test_dependency\" > $depfile\n" +"build out: generate-depfile\n" +" test_dependency = inimp\n" +" depfile = out.d\n")); + fs_.Create("inimp", ""); + fs_.Tick(); + + string err; + + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_FALSE(builder_.AlreadyUpToDate()); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_TRUE(builder_.AlreadyUpToDate()); + + command_runner_.commands_ran_.clear(); + state_.Reset(); + builder_.Cleanup(); + builder_.plan_.Reset(); + + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_TRUE(builder_.AlreadyUpToDate()); +} + struct BuildDryRun : public BuildWithLogTest { BuildDryRun() { config_.dry_run = true; @@ -1587,7 +2021,30 @@ TEST_F(BuildDryRun, AllCommandsShown) { string err; EXPECT_TRUE(builder_.AddTarget("out3", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); +} + +TEST_F(BuildDryRun, WithDyndep) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build out: touch || dd\n" +" dyndep = dd\n" +"build out-copy: cp out\n" +)); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out: dyndep\n" +); + + string err; + EXPECT_TRUE(builder_.AddTarget("out-copy", &err)); + ASSERT_EQ("", err); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); } @@ -1630,13 +2087,14 @@ TEST_F(BuildTest, RspFileSuccess) size_t files_created = fs_.files_created_.size(); size_t files_removed = fs_.files_removed_.size(); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); - // The RSP files were created - ASSERT_EQ(files_created + 2, fs_.files_created_.size()); + // The RSP files and temp file to acquire output mtimes were created + ASSERT_EQ(files_created + 3, fs_.files_created_.size()); ASSERT_EQ(1u, fs_.files_created_.count("out 2.rsp")); ASSERT_EQ(1u, fs_.files_created_.count("out 3.rsp")); + ASSERT_EQ(1u, fs_.files_created_.count(".ninja_lock")); // The RSP files were removed ASSERT_EQ(files_removed + 2, fs_.files_removed_.size()); @@ -1666,13 +2124,14 @@ TEST_F(BuildTest, RspFileFailure) { size_t files_created = fs_.files_created_.size(); size_t files_removed = fs_.files_removed_.size(); - EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitFailure); ASSERT_EQ("subcommand failed", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); - // The RSP file was created - ASSERT_EQ(files_created + 1, fs_.files_created_.size()); + // The RSP file and temp file to acquire output mtimes were created + ASSERT_EQ(files_created + 2, fs_.files_created_.size()); ASSERT_EQ(1u, fs_.files_created_.count("out.rsp")); + ASSERT_EQ(1u, fs_.files_created_.count(".ninja_lock")); // The RSP file was NOT removed ASSERT_EQ(files_removed, fs_.files_removed_.size()); @@ -1703,7 +2162,7 @@ TEST_F(BuildWithLogTest, RspFileCmdLineChange) { ASSERT_EQ("", err); // 1. Build for the 1st time (-> populate log) - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); // 2. Build again (no change) @@ -1726,7 +2185,7 @@ TEST_F(BuildWithLogTest, RspFileCmdLineChange) { state_.Reset(); EXPECT_TRUE(builder_.AddTarget("out", &err)); EXPECT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ(1u, command_runner_.commands_ran_.size()); } @@ -1749,7 +2208,7 @@ TEST_F(BuildTest, InterruptCleanup) { string err; EXPECT_TRUE(builder_.AddTarget("out1", &err)); EXPECT_EQ("", err); - EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitInterrupted); EXPECT_EQ("interrupted by user", err); builder_.Cleanup(); EXPECT_GT(fs_.Stat("out1", &err), 0); @@ -1758,7 +2217,7 @@ TEST_F(BuildTest, InterruptCleanup) { // A touched output of an interrupted command should be deleted. EXPECT_TRUE(builder_.AddTarget("out2", &err)); EXPECT_EQ("", err); - EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitInterrupted); EXPECT_EQ("interrupted by user", err); builder_.Cleanup(); EXPECT_EQ(0, fs_.Stat("out2", &err)); @@ -1800,7 +2259,7 @@ TEST_F(BuildTest, PhonyWithNoInputs) { state_.Reset(); EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); } @@ -1818,23 +2277,38 @@ TEST_F(BuildTest, DepsGccWithEmptyDepfileErrorsOut) { ASSERT_EQ("", err); EXPECT_FALSE(builder_.AlreadyUpToDate()); - EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitFailure); ASSERT_EQ("subcommand failed", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); } -TEST_F(BuildTest, StatusFormatElapsed) { +TEST_F(BuildTest, StatusFormatElapsed_e) { + status_.BuildStarted(); + // Before any task is done, the elapsed time must be zero. + EXPECT_EQ("[%/e0.000]", status_.FormatProgressStatus("[%%/e%e]", 0)); +} + +TEST_F(BuildTest, StatusFormatElapsed_w) { status_.BuildStarted(); // Before any task is done, the elapsed time must be zero. - EXPECT_EQ("[%/e0.000]", - status_.FormatProgressStatus("[%%/e%e]", - BuildStatus::kEdgeStarted)); + EXPECT_EQ("[%/e00:00]", status_.FormatProgressStatus("[%%/e%w]", 0)); +} + +TEST_F(BuildTest, StatusFormatETA) { + status_.BuildStarted(); + // Before any task is done, the ETA time must be unknown. + EXPECT_EQ("[%/E?]", status_.FormatProgressStatus("[%%/E%E]", 0)); +} + +TEST_F(BuildTest, StatusFormatTimeProgress) { + status_.BuildStarted(); + // Before any task is done, the percentage of elapsed time must be zero. + EXPECT_EQ("[%/p 0%]", status_.FormatProgressStatus("[%%/p%p]", 0)); } TEST_F(BuildTest, StatusFormatReplacePlaceholder) { EXPECT_EQ("[%/s0/t0/r0/u0/f0]", - status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]", - BuildStatus::kEdgeStarted)); + status_.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]", 0)); } TEST_F(BuildTest, FailedDepsParse) { @@ -1851,16 +2325,226 @@ TEST_F(BuildTest, FailedDepsParse) { // path to the left of the colon. fs_.Create("in1.d", "AAA BBB"); - EXPECT_FALSE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitFailure); EXPECT_EQ("subcommand failed", err); } -/// Tests of builds involving deps logs necessarily must span +struct BuildWithQueryDepsLogTest : public BuildTest { + BuildWithQueryDepsLogTest() + : BuildTest(&log_), deps_log_file_("ninja_deps") {} + + ~BuildWithQueryDepsLogTest() { + log_.Close(); + } + + virtual void SetUp() { + BuildTest::SetUp(); + + temp_dir_.CreateAndEnter("BuildWithQueryDepsLogTest"); + + std::string err; + ASSERT_TRUE(log_.OpenForWrite(deps_log_file_.path(), &err)); + ASSERT_EQ("", err); + } + + ScopedTempDir temp_dir_; + + ScopedFilePath deps_log_file_; + DepsLog log_; +}; + +/// Test a MSVC-style deps log with multiple outputs. +TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileMSVC) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cp_multi_msvc\n" +" command = echo 'using $in' && for file in $out; do cp $in $$file; done\n" +" deps = msvc\n" +" msvc_deps_prefix = using \n" +"build out1 out2: cp_multi_msvc in1\n")); + + std::string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("echo 'using in1' && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]); + + Node* out1_node = state_.LookupNode("out1"); + DepsLog::Deps* out1_deps = log_.GetDeps(out1_node); + EXPECT_EQ(1, out1_deps->node_count); + EXPECT_EQ("in1", out1_deps->nodes[0]->path()); + + Node* out2_node = state_.LookupNode("out2"); + DepsLog::Deps* out2_deps = log_.GetDeps(out2_node); + EXPECT_EQ(1, out2_deps->node_count); + EXPECT_EQ("in1", out2_deps->nodes[0]->path()); +} + +/// Test a GCC-style deps log with multiple outputs. +TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCOneLine) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cp_multi_gcc\n" +" command = echo '$out: $in' > in.d && for file in $out; do cp in1 $$file; done\n" +" deps = gcc\n" +" depfile = in.d\n" +"build out1 out2: cp_multi_gcc in1 in2\n")); + + std::string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + fs_.Create("in.d", "out1 out2: in1 in2"); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("echo 'out1 out2: in1 in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]); + + Node* out1_node = state_.LookupNode("out1"); + DepsLog::Deps* out1_deps = log_.GetDeps(out1_node); + EXPECT_EQ(2, out1_deps->node_count); + EXPECT_EQ("in1", out1_deps->nodes[0]->path()); + EXPECT_EQ("in2", out1_deps->nodes[1]->path()); + + Node* out2_node = state_.LookupNode("out2"); + DepsLog::Deps* out2_deps = log_.GetDeps(out2_node); + EXPECT_EQ(2, out2_deps->node_count); + EXPECT_EQ("in1", out2_deps->nodes[0]->path()); + EXPECT_EQ("in2", out2_deps->nodes[1]->path()); +} + +/// Test a GCC-style deps log with multiple outputs using a line per input. +TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCMultiLineInput) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cp_multi_gcc\n" +" command = echo '$out: in1\\n$out: in2' > in.d && for file in $out; do cp in1 $$file; done\n" +" deps = gcc\n" +" depfile = in.d\n" +"build out1 out2: cp_multi_gcc in1 in2\n")); + + std::string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + fs_.Create("in.d", "out1 out2: in1\nout1 out2: in2"); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("echo 'out1 out2: in1\\nout1 out2: in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]); + + Node* out1_node = state_.LookupNode("out1"); + DepsLog::Deps* out1_deps = log_.GetDeps(out1_node); + EXPECT_EQ(2, out1_deps->node_count); + EXPECT_EQ("in1", out1_deps->nodes[0]->path()); + EXPECT_EQ("in2", out1_deps->nodes[1]->path()); + + Node* out2_node = state_.LookupNode("out2"); + DepsLog::Deps* out2_deps = log_.GetDeps(out2_node); + EXPECT_EQ(2, out2_deps->node_count); + EXPECT_EQ("in1", out2_deps->nodes[0]->path()); + EXPECT_EQ("in2", out2_deps->nodes[1]->path()); +} + +/// Test a GCC-style deps log with multiple outputs using a line per output. +TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCMultiLineOutput) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cp_multi_gcc\n" +" command = echo 'out1: $in\\nout2: $in' > in.d && for file in $out; do cp in1 $$file; done\n" +" deps = gcc\n" +" depfile = in.d\n" +"build out1 out2: cp_multi_gcc in1 in2\n")); + + std::string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + fs_.Create("in.d", "out1: in1 in2\nout2: in1 in2"); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("echo 'out1: in1 in2\\nout2: in1 in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]); + + Node* out1_node = state_.LookupNode("out1"); + DepsLog::Deps* out1_deps = log_.GetDeps(out1_node); + EXPECT_EQ(2, out1_deps->node_count); + EXPECT_EQ("in1", out1_deps->nodes[0]->path()); + EXPECT_EQ("in2", out1_deps->nodes[1]->path()); + + Node* out2_node = state_.LookupNode("out2"); + DepsLog::Deps* out2_deps = log_.GetDeps(out2_node); + EXPECT_EQ(2, out2_deps->node_count); + EXPECT_EQ("in1", out2_deps->nodes[0]->path()); + EXPECT_EQ("in2", out2_deps->nodes[1]->path()); +} + +/// Test a GCC-style deps log with multiple outputs mentioning only the main output. +TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCOnlyMainOutput) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cp_multi_gcc\n" +" command = echo 'out1: $in' > in.d && for file in $out; do cp in1 $$file; done\n" +" deps = gcc\n" +" depfile = in.d\n" +"build out1 out2: cp_multi_gcc in1 in2\n")); + + std::string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + fs_.Create("in.d", "out1: in1 in2"); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("echo 'out1: in1 in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]); + + Node* out1_node = state_.LookupNode("out1"); + DepsLog::Deps* out1_deps = log_.GetDeps(out1_node); + EXPECT_EQ(2, out1_deps->node_count); + EXPECT_EQ("in1", out1_deps->nodes[0]->path()); + EXPECT_EQ("in2", out1_deps->nodes[1]->path()); + + Node* out2_node = state_.LookupNode("out2"); + DepsLog::Deps* out2_deps = log_.GetDeps(out2_node); + EXPECT_EQ(2, out2_deps->node_count); + EXPECT_EQ("in1", out2_deps->nodes[0]->path()); + EXPECT_EQ("in2", out2_deps->nodes[1]->path()); +} + +/// Test a GCC-style deps log with multiple outputs mentioning only the secondary output. +TEST_F(BuildWithQueryDepsLogTest, TwoOutputsDepFileGCCOnlySecondaryOutput) { + // Note: This ends up short-circuiting the node creation due to the primary + // output not being present, but it should still work. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule cp_multi_gcc\n" +" command = echo 'out2: $in' > in.d && for file in $out; do cp in1 $$file; done\n" +" deps = gcc\n" +" depfile = in.d\n" +"build out1 out2: cp_multi_gcc in1 in2\n")); + + std::string err; + EXPECT_TRUE(builder_.AddTarget("out1", &err)); + ASSERT_EQ("", err); + fs_.Create("in.d", "out2: in1 in2"); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("echo 'out2: in1 in2' > in.d && for file in out1 out2; do cp in1 $file; done", command_runner_.commands_ran_[0]); + + Node* out1_node = state_.LookupNode("out1"); + DepsLog::Deps* out1_deps = log_.GetDeps(out1_node); + EXPECT_EQ(2, out1_deps->node_count); + EXPECT_EQ("in1", out1_deps->nodes[0]->path()); + EXPECT_EQ("in2", out1_deps->nodes[1]->path()); + + Node* out2_node = state_.LookupNode("out2"); + DepsLog::Deps* out2_deps = log_.GetDeps(out2_node); + EXPECT_EQ(2, out2_deps->node_count); + EXPECT_EQ("in1", out2_deps->nodes[0]->path()); + EXPECT_EQ("in2", out2_deps->nodes[1]->path()); +} + +/// Tests of builds involving deps logs necessarily must span /// multiple builds. We reuse methods on BuildTest but not the /// builder_ it sets up, because we want pristine objects for /// each build. struct BuildWithDepsLogTest : public BuildTest { - BuildWithDepsLogTest() {} + BuildWithDepsLogTest() + : build_log_file_("build_log"), deps_log_file_("ninja_deps") {} virtual void SetUp() { BuildTest::SetUp(); @@ -1873,12 +2557,14 @@ struct BuildWithDepsLogTest : public BuildTest { } ScopedTempDir temp_dir_; + ScopedFilePath build_log_file_; + ScopedFilePath deps_log_file_; /// Shadow parent class builder_ so we don't accidentally use it. void* builder_; }; -/// Run a straightforwad build where the deps log is used. +/// Run a straightforward build where the deps log is used. TEST_F(BuildWithDepsLogTest, Straightforward) { string err; // Note: in1 was created by the superclass SetUp(). @@ -1886,6 +2572,7 @@ TEST_F(BuildWithDepsLogTest, Straightforward) { "build out: cat in1\n" " deps = gcc\n" " depfile = in1.d\n"; + { State state; ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); @@ -1893,15 +2580,15 @@ TEST_F(BuildWithDepsLogTest, Straightforward) { // Run the build once, everything should be ok. DepsLog deps_log; - ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); fs_.Create("in1.d", "out: in2"); - EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ(builder.Build(&err), ExitSuccess); EXPECT_EQ("", err); // The deps file should have been removed. @@ -1923,15 +2610,15 @@ TEST_F(BuildWithDepsLogTest, Straightforward) { // Run the build again. DepsLog deps_log; - ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); - ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_TRUE(deps_log.Load(deps_log_file_.path(), &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ(builder.Build(&err), ExitSuccess); EXPECT_EQ("", err); // We should have rebuilt the output due to in2 being @@ -1964,14 +2651,14 @@ TEST_F(BuildWithDepsLogTest, ObsoleteDeps) { // Run the build once, everything should be ok. DepsLog deps_log; - ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ(builder.Build(&err), ExitSuccess); EXPECT_EQ("", err); deps_log.Close(); @@ -1993,10 +2680,10 @@ TEST_F(BuildWithDepsLogTest, ObsoleteDeps) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); DepsLog deps_log; - ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); - ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_TRUE(deps_log.Load(deps_log_file_.path(), &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); EXPECT_TRUE(builder.AddTarget("out", &err)); @@ -2005,7 +2692,7 @@ TEST_F(BuildWithDepsLogTest, ObsoleteDeps) { // Recreate the deps file here because the build expects them to exist. fs_.Create("in1.d", "out: "); - EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ(builder.Build(&err), ExitSuccess); EXPECT_EQ("", err); // We should have rebuilt the output due to the deps being @@ -2032,19 +2719,223 @@ TEST_F(BuildWithDepsLogTest, DepsIgnoredInDryRun) { // The deps log is NULL in dry runs. config_.dry_run = true; - Builder builder(&state, config_, NULL, NULL, &fs_); + Builder builder(&state, config_, NULL, NULL, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); string err; EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ(builder.Build(&err), ExitSuccess); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); builder.command_runner_.release(); } +TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceCondition) { + string err; + const char* manifest = + "rule long-cc\n" + " command = long-cc\n" + "build out: long-cc in1\n" + " test_dependency = in1\n"; + + State state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + BuildLog build_log; + ASSERT_TRUE(build_log.Load(build_log_file_.path(), &err)); + ASSERT_TRUE(build_log.OpenForWrite(build_log_file_.path(), *this, &err)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load(deps_log_file_.path(), &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); + + BuildLog::LogEntry* log_entry = NULL; + { + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + + // Run the build, out gets built, dep file is created + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_EQ(builder.Build(&err), ExitSuccess); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + + // See that an entry in the logfile is created. the input_mtime is 1 since that was + // the mtime of in1 when the command was started + log_entry = build_log.LookupByOutput("out"); + ASSERT_TRUE(NULL != log_entry); + ASSERT_EQ(1u, log_entry->mtime); + + builder.command_runner_.release(); + } + + { + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + + // Trigger the build again - "out" should rebuild despite having a newer mtime than + // "in1", since "in1" was touched during the build of out (simulated by changing its + // mtime in the the test builder's WaitForCommand() which runs before FinishCommand() + command_runner_.commands_ran_.clear(); + state.Reset(); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_EQ(builder.Build(&err), ExitSuccess); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + + // Check that the logfile entry is still correct + log_entry = build_log.LookupByOutput("out"); + ASSERT_TRUE(NULL != log_entry); + ASSERT_TRUE(fs_.files_["in1"].mtime < log_entry->mtime); + builder.command_runner_.release(); + } + + { + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + + // And a subsequent run should not have any work to do + command_runner_.commands_ran_.clear(); + state.Reset(); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder.AlreadyUpToDate()); + + builder.command_runner_.release(); + } +} + +TEST_F(BuildWithDepsLogTest, TestInputMtimeRaceConditionWithDepFile) { + string err; + const char* manifest = + "rule long-cc\n" + " command = long-cc\n" + "build out: long-cc\n" + " deps = gcc\n" + " depfile = out.d\n" + " test_dependency = header.h\n"; + + fs_.Create("header.h", ""); + + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + BuildLog build_log; + ASSERT_TRUE(build_log.Load(build_log_file_.path(), &err)); + ASSERT_TRUE(build_log.OpenForWrite(build_log_file_.path(), *this, &err)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load(deps_log_file_.path(), &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); + + { + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + + // Run the build, out gets built, dep file is created + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_EQ(builder.Build(&err), ExitSuccess); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + + // See that an entry in the logfile is created. the mtime is 1 due to the command + // starting when the file system's mtime was 1. + BuildLog::LogEntry* log_entry = build_log.LookupByOutput("out"); + ASSERT_TRUE(NULL != log_entry); + ASSERT_EQ(1u, log_entry->mtime); + + builder.command_runner_.release(); + } + + { + // Trigger the build again - "out" will rebuild since its newest input mtime (header.h) + // is newer than the recorded mtime of out in the build log + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + + state.Reset(); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_EQ(builder.Build(&err), ExitSuccess); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + + builder.command_runner_.release(); + } + + { + // Trigger the build again - "out" won't rebuild since the file wasn't updated during + // the previous build + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + + state.Reset(); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + ASSERT_TRUE(builder.AlreadyUpToDate()); + + builder.command_runner_.release(); + } + + // touch the header to trigger a rebuild + fs_.Create("header.h", ""); + ASSERT_EQ(fs_.now_, 7); + + { + // Rebuild. This time, long-cc will cause header.h to be updated while the build is + // in progress + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + + state.Reset(); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_EQ(builder.Build(&err), ExitSuccess); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + + builder.command_runner_.release(); + } + + { + // Rebuild. Because header.h is now in the deplog for out, it should be detectable as + // a change-while-in-progress and should cause a rebuild of out. + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + + state.Reset(); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_EQ(builder.Build(&err), ExitSuccess); + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + + builder.command_runner_.release(); + } + + { + // This time, the header.h file was not updated during the build, so the target should + // not be considered dirty. + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + command_runner_.commands_ran_.clear(); + + state.Reset(); + EXPECT_TRUE(builder.AddTarget("out", &err)); + ASSERT_EQ("", err); + EXPECT_TRUE(builder.AlreadyUpToDate()); + + builder.command_runner_.release(); + } +} + /// Check that a restat rule generating a header cancels compilations correctly. TEST_F(BuildTest, RestatDepfileDependency) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, @@ -2063,7 +2954,7 @@ TEST_F(BuildTest, RestatDepfileDependency) { string err; EXPECT_TRUE(builder_.AddTarget("out", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); } @@ -2087,15 +2978,15 @@ TEST_F(BuildWithDepsLogTest, RestatDepfileDependencyDepsLog) { // Run the build once, everything should be ok. DepsLog deps_log; - ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); fs_.Create("in1.d", "out: header.h"); - EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ(builder.Build(&err), ExitSuccess); EXPECT_EQ("", err); deps_log.Close(); @@ -2113,15 +3004,15 @@ TEST_F(BuildWithDepsLogTest, RestatDepfileDependencyDepsLog) { // Run the build again. DepsLog deps_log; - ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); - ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_TRUE(deps_log.Load(deps_log_file_.path(), &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); command_runner_.commands_ran_.clear(); EXPECT_TRUE(builder.AddTarget("out", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ(builder.Build(&err), ExitSuccess); EXPECT_EQ("", err); // Rule "true" should have run again, but the build of "out" should have @@ -2146,15 +3037,15 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { // Run the build once, everything should be ok. DepsLog deps_log; - ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("fo o.o", &err)); ASSERT_EQ("", err); fs_.Create("fo o.o.d", "fo\\ o.o: blah.h bar.h\n"); - EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ(builder.Build(&err), ExitSuccess); EXPECT_EQ("", err); deps_log.Close(); @@ -2166,11 +3057,11 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); DepsLog deps_log; - ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); - ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_TRUE(deps_log.Load(deps_log_file_.path(), &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); Edge* edge = state.edges_.back(); @@ -2179,9 +3070,9 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { EXPECT_TRUE(builder.AddTarget("fo o.o", &err)); ASSERT_EQ("", err); - // Expect three new edges: one generating fo o.o, and two more from - // loading the depfile. - ASSERT_EQ(3u, state.edges_.size()); + // Expect one new edge generating fo o.o, loading the depfile should + // not generate new edges. + ASSERT_EQ(1u, state.edges_.size()); // Expect our edge to now have three inputs: foo.c and two headers. ASSERT_EQ(3u, edge->inputs_.size()); @@ -2193,6 +3084,90 @@ TEST_F(BuildWithDepsLogTest, DepFileOKDepsLog) { } } +TEST_F(BuildWithDepsLogTest, DiscoveredDepDuringBuildChanged) { + string err; + const char* manifest = + "rule touch-out-implicit-dep\n" + " command = touch $out ; sleep 1 ; touch $test_dependency\n" + "rule generate-depfile\n" + " command = touch $out ; echo \"$out: $test_dependency\" > $depfile\n" + "build out1: touch-out-implicit-dep in1\n" + " test_dependency = inimp\n" + "build out2: generate-depfile in1 || out1\n" + " test_dependency = inimp\n" + " depfile = out2.d\n" + " deps = gcc\n"; + + fs_.Create("in1", ""); + fs_.Tick(); + + BuildLog build_log; + + { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + EXPECT_TRUE(builder.AddTarget("out2", &err)); + EXPECT_FALSE(builder.AlreadyUpToDate()); + + EXPECT_EQ(builder.Build(&err), ExitSuccess); + EXPECT_TRUE(builder.AlreadyUpToDate()); + + deps_log.Close(); + builder.command_runner_.release(); + } + + fs_.Tick(); + fs_.Create("in1", ""); + + { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load(deps_log_file_.path(), &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + EXPECT_TRUE(builder.AddTarget("out2", &err)); + EXPECT_FALSE(builder.AlreadyUpToDate()); + + EXPECT_EQ(builder.Build(&err), ExitSuccess); + EXPECT_TRUE(builder.AlreadyUpToDate()); + + deps_log.Close(); + builder.command_runner_.release(); + } + + fs_.Tick(); + + { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load(deps_log_file_.path(), &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, &build_log, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + EXPECT_TRUE(builder.AddTarget("out2", &err)); + EXPECT_TRUE(builder.AlreadyUpToDate()); + + deps_log.Close(); + builder.command_runner_.release(); + } +} + #ifdef _WIN32 TEST_F(BuildWithDepsLogTest, DepFileDepsLogCanonicalize) { string err; @@ -2208,17 +3183,17 @@ TEST_F(BuildWithDepsLogTest, DepFileDepsLogCanonicalize) { // Run the build once, everything should be ok. DepsLog deps_log; - ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); EXPECT_TRUE(builder.AddTarget("a/b/c/d/e/fo o.o", &err)); ASSERT_EQ("", err); // Note, different slashes from manifest. fs_.Create("a/b\\c\\d/e/fo o.o.d", "a\\b\\c\\d\\e\\fo\\ o.o: blah.h bar.h\n"); - EXPECT_TRUE(builder.Build(&err)); + EXPECT_EQ(builder.Build(&err), ExitSuccess); EXPECT_EQ("", err); deps_log.Close(); @@ -2230,23 +3205,21 @@ TEST_F(BuildWithDepsLogTest, DepFileDepsLogCanonicalize) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); DepsLog deps_log; - ASSERT_TRUE(deps_log.Load("ninja_deps", &state, &err)); - ASSERT_TRUE(deps_log.OpenForWrite("ninja_deps", &err)); + ASSERT_TRUE(deps_log.Load(deps_log_file_.path(), &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); ASSERT_EQ("", err); - Builder builder(&state, config_, NULL, &deps_log, &fs_); + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); builder.command_runner_.reset(&command_runner_); - Edge* edge = state.edges_.back(); - state.GetNode("bar.h", 0)->MarkDirty(); // Mark bar.h as missing. EXPECT_TRUE(builder.AddTarget("a/b/c/d/e/fo o.o", &err)); ASSERT_EQ("", err); - // Expect three new edges: one generating fo o.o, and two more from - // loading the depfile. - ASSERT_EQ(3u, state.edges_.size()); + // Expect one new edge generating fo o.o. + ASSERT_EQ(1u, state.edges_.size()); // Expect our edge to now have three inputs: foo.c and two headers. + Edge* edge = state.edges_.back(); ASSERT_EQ(3u, edge->inputs_.size()); // Expect the command line we generate to only use the original input. @@ -2301,25 +3274,29 @@ TEST_F(BuildWithDepsLogTest, RestatMissingDepfileDepslog) { fs_.Create("out.d", "out: header.h"); fs_.Create("header.h", ""); - RebuildTarget("out", manifest, "build_log", "ninja_deps"); + RebuildTarget("out", manifest, build_log_file_.c_str(), + deps_log_file_.c_str()); ASSERT_EQ(2u, command_runner_.commands_ran_.size()); // Sanity: this rebuild should be NOOP - RebuildTarget("out", manifest, "build_log", "ninja_deps"); + RebuildTarget("out", manifest, build_log_file_.c_str(), + deps_log_file_.c_str()); ASSERT_EQ(0u, command_runner_.commands_ran_.size()); // Touch 'header.in', blank dependencies log (create a different one). // Building header.h triggers 'restat' outputs cleanup. - // Validate that out is rebuilt netherless, as deps are missing. + // Validate that out is rebuilt nevertheless, as deps are missing. fs_.Tick(); fs_.Create("header.in", ""); + ScopedFilePath deps2_file_("ninja_deps2"); + // (switch to a new blank deps_log "ninja_deps2") - RebuildTarget("out", manifest, "build_log", "ninja_deps2"); + RebuildTarget("out", manifest, build_log_file_.c_str(), deps2_file_.c_str()); ASSERT_EQ(2u, command_runner_.commands_ran_.size()); // Sanity: this build should be NOOP - RebuildTarget("out", manifest, "build_log", "ninja_deps2"); + RebuildTarget("out", manifest, build_log_file_.c_str(), deps2_file_.c_str()); ASSERT_EQ(0u, command_runner_.commands_ran_.size()); // Check that invalidating deps by target timestamp also works here @@ -2327,11 +3304,11 @@ TEST_F(BuildWithDepsLogTest, RestatMissingDepfileDepslog) { fs_.Tick(); fs_.Create("header.in", ""); fs_.Create("out", ""); - RebuildTarget("out", manifest, "build_log", "ninja_deps2"); + RebuildTarget("out", manifest, build_log_file_.c_str(), deps2_file_.c_str()); ASSERT_EQ(2u, command_runner_.commands_ran_.size()); // And this build should be NOOP again - RebuildTarget("out", manifest, "build_log", "ninja_deps2"); + RebuildTarget("out", manifest, build_log_file_.c_str(), deps2_file_.c_str()); ASSERT_EQ(0u, command_runner_.commands_ran_.size()); } @@ -2348,7 +3325,10 @@ TEST_F(BuildTest, WrongOutputInDepfileCausesRebuild) { fs_.Create("header.h", ""); fs_.Create("foo.o.d", "bar.o.d: header.h\n"); - RebuildTarget("foo.o", manifest, "build_log", "ninja_deps"); + ScopedFilePath build_log("build_log"); + ScopedFilePath deps_file("ninja_deps"); + + RebuildTarget("foo.o", manifest, build_log.c_str(), deps_file.c_str()); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); } @@ -2364,7 +3344,7 @@ TEST_F(BuildTest, Console) { string err; EXPECT_TRUE(builder_.AddTarget("cons", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); } @@ -2405,7 +3385,7 @@ TEST_F(BuildTest, DyndepReadyImplicitConnection) { string err; EXPECT_TRUE(builder_.AddTarget("out", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(2u, command_runner_.commands_ran_.size()); EXPECT_EQ("touch tmp tmp.imp", command_runner_.commands_ran_[0]); @@ -2472,7 +3452,7 @@ TEST_F(BuildTest, DyndepBuild) { EXPECT_EQ("", err); size_t files_created = fs_.files_created_.size(); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(2u, command_runner_.commands_ran_.size()); @@ -2481,9 +3461,10 @@ TEST_F(BuildTest, DyndepBuild) { ASSERT_EQ(2u, fs_.files_read_.size()); EXPECT_EQ("dd-in", fs_.files_read_[0]); EXPECT_EQ("dd", fs_.files_read_[1]); - ASSERT_EQ(2u + files_created, fs_.files_created_.size()); + ASSERT_EQ(3u + files_created, fs_.files_created_.size()); EXPECT_EQ(1u, fs_.files_created_.count("dd")); EXPECT_EQ(1u, fs_.files_created_.count("out")); + EXPECT_EQ(1u, fs_.files_created_.count(".ninja_lock")); } TEST_F(BuildTest, DyndepBuildSyntaxError) { @@ -2534,7 +3515,7 @@ TEST_F(BuildTest, DyndepBuildUnrelatedOutput) { EXPECT_TRUE(builder_.AddTarget("out", &err)); EXPECT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); @@ -2566,7 +3547,7 @@ TEST_F(BuildTest, DyndepBuildDiscoverNewOutput) { EXPECT_TRUE(builder_.AddTarget("out", &err)); EXPECT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(2u, command_runner_.commands_ran_.size()); EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); @@ -2668,7 +3649,7 @@ TEST_F(BuildTest, DyndepBuildDiscoverNewInput) { EXPECT_TRUE(builder_.AddTarget("out", &err)); EXPECT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); @@ -2676,6 +3657,67 @@ TEST_F(BuildTest, DyndepBuildDiscoverNewInput) { EXPECT_EQ("touch out", command_runner_.commands_ran_[2]); } +TEST_F(BuildTest, DyndepBuildDiscoverNewInputWithValidation) { + // Verify that a dyndep file cannot contain the |@ validation + // syntax. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build out: touch || dd\n" +" dyndep = dd\n" +)); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out: dyndep |@ validation\n" +); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_FALSE(builder_.Build(&err)); + + string err_first_line = err.substr(0, err.find("\n")); + EXPECT_EQ("dd:2: expected newline, got '|@'", err_first_line); +} + +TEST_F(BuildTest, DyndepBuildDiscoverNewInputWithTransitiveValidation) { + // Verify that a dyndep file can be built and loaded to discover + // a new input to an edge that has a validation edge. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build in: touch |@ validation\n" +"build validation: touch in out\n" +"build out: touch || dd\n" +" dyndep = dd\n" + )); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build out: dyndep | in\n" +); + fs_.Tick(); + fs_.Create("out", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + ASSERT_EQ(4u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch in", command_runner_.commands_ran_[1]); + EXPECT_EQ("touch out", command_runner_.commands_ran_[2]); + EXPECT_EQ("touch validation", command_runner_.commands_ran_[3]); +} + TEST_F(BuildTest, DyndepBuildDiscoverImplicitConnection) { // Verify that a dyndep file can be built and loaded to discover // that one edge has an implicit output that is also an implicit @@ -2700,7 +3742,7 @@ TEST_F(BuildTest, DyndepBuildDiscoverImplicitConnection) { string err; EXPECT_TRUE(builder_.AddTarget("out", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); @@ -2708,6 +3750,48 @@ TEST_F(BuildTest, DyndepBuildDiscoverImplicitConnection) { EXPECT_EQ("touch out out.imp", command_runner_.commands_ran_[2]); } +TEST_F(BuildTest, DyndepBuildDiscoverOutputAndDepfileInput) { + // Verify that a dyndep file can be built and loaded to discover + // that one edge has an implicit output that is also reported by + // a depfile as an input of another edge. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out $out.imp\n" +"rule cp\n" +" command = cp $in $out\n" +"build dd: cp dd-in\n" +"build tmp: touch || dd\n" +" dyndep = dd\n" +"build out: cp tmp\n" +" depfile = out.d\n" +)); + fs_.Create("out.d", "out: tmp.imp\n"); + fs_.Create("dd-in", +"ninja_dyndep_version = 1\n" +"build tmp | tmp.imp: dyndep\n" +); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + // Loading the depfile did not give tmp.imp a phony input edge. + ASSERT_FALSE(GetNode("tmp.imp")->in_edge()); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + + // Loading the dyndep file gave tmp.imp a real input edge. + ASSERT_FALSE(GetNode("tmp.imp")->in_edge()->is_phony()); + + ASSERT_EQ(3u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); + EXPECT_EQ("touch tmp tmp.imp", command_runner_.commands_ran_[1]); + EXPECT_EQ("cp tmp out", command_runner_.commands_ran_[2]); + EXPECT_EQ(1u, fs_.files_created_.count("tmp.imp")); + EXPECT_TRUE(builder_.AlreadyUpToDate()); +} + TEST_F(BuildTest, DyndepBuildDiscoverNowWantEdge) { // Verify that a dyndep file can be built and loaded to discover // that an edge is actually wanted due to a missing implicit output. @@ -2733,7 +3817,7 @@ TEST_F(BuildTest, DyndepBuildDiscoverNowWantEdge) { string err; EXPECT_TRUE(builder_.AddTarget("out", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); @@ -2764,7 +3848,7 @@ TEST_F(BuildTest, DyndepBuildDiscoverNowWantEdgeAndDependent) { string err; EXPECT_TRUE(builder_.AddTarget("out", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); @@ -2835,7 +3919,7 @@ TEST_F(BuildWithLogTest, DyndepBuildDiscoverRestat) { string err; EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ("", err); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); EXPECT_EQ("cp dd-in dd", command_runner_.commands_ran_[0]); @@ -2851,7 +3935,7 @@ TEST_F(BuildWithLogTest, DyndepBuildDiscoverRestat) { // touch "out1", we should cancel the build of "out2". EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); ASSERT_EQ(1u, command_runner_.commands_ran_.size()); EXPECT_EQ("true", command_runner_.commands_ran_[0]); } @@ -2892,7 +3976,7 @@ TEST_F(BuildTest, DyndepBuildDiscoverScheduledEdge) { EXPECT_TRUE(builder_.AddTarget("out1", &err)); EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); // Depending on how the pointers in Plan::ready_ work out, the first @@ -2943,7 +4027,7 @@ TEST_F(BuildTest, DyndepTwoLevelDirect) { string err; EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); EXPECT_EQ("cp dd1-in dd1", command_runner_.commands_ran_[0]); @@ -2988,7 +4072,7 @@ TEST_F(BuildTest, DyndepTwoLevelIndirect) { string err; EXPECT_TRUE(builder_.AddTarget("out2", &err)); ASSERT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(3u, command_runner_.commands_ran_.size()); EXPECT_EQ("cp dd1-in dd1", command_runner_.commands_ran_[0]); @@ -3028,7 +4112,7 @@ TEST_F(BuildTest, DyndepTwoLevelDiscoveredReady) { EXPECT_TRUE(builder_.AddTarget("out", &err)); EXPECT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(4u, command_runner_.commands_ran_.size()); EXPECT_EQ("cp dd1-in dd1", command_runner_.commands_ran_[0]); @@ -3068,7 +4152,7 @@ TEST_F(BuildTest, DyndepTwoLevelDiscoveredDirty) { EXPECT_TRUE(builder_.AddTarget("out", &err)); EXPECT_EQ("", err); - EXPECT_TRUE(builder_.Build(&err)); + EXPECT_EQ(builder_.Build(&err), ExitSuccess); EXPECT_EQ("", err); ASSERT_EQ(5u, command_runner_.commands_ran_.size()); EXPECT_EQ("cp dd1-in dd1", command_runner_.commands_ran_[0]); @@ -3077,3 +4161,247 @@ TEST_F(BuildTest, DyndepTwoLevelDiscoveredDirty) { EXPECT_EQ("touch tmp", command_runner_.commands_ran_[3]); EXPECT_EQ("touch out", command_runner_.commands_ran_[4]); } + +TEST_F(BuildTest, Validation) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out: cat in |@ validate\n" + "build validate: cat in2\n")); + + fs_.Create("in", ""); + fs_.Create("in2", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + + EXPECT_EQ(2u, command_runner_.commands_ran_.size()); + + // Test touching "in" only rebuilds "out" ("validate" doesn't depend on + // "out"). + fs_.Tick(); + fs_.Create("in", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in > out", command_runner_.commands_ran_[0]); + + // Test touching "in2" only rebuilds "validate" ("out" doesn't depend on + // "validate"). + fs_.Tick(); + fs_.Create("in2", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in2 > validate", command_runner_.commands_ran_[0]); +} + +TEST_F(BuildTest, ValidationDependsOnOutput) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out: cat in |@ validate\n" + "build validate: cat in2 | out\n")); + + fs_.Create("in", ""); + fs_.Create("in2", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + + EXPECT_EQ(2u, command_runner_.commands_ran_.size()); + + // Test touching "in" rebuilds "out" and "validate". + fs_.Tick(); + fs_.Create("in", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + + EXPECT_EQ(2u, command_runner_.commands_ran_.size()); + + // Test touching "in2" only rebuilds "validate" ("out" doesn't depend on + // "validate"). + fs_.Tick(); + fs_.Create("in2", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in2 > validate", command_runner_.commands_ran_[0]); +} + +TEST_F(BuildWithDepsLogTest, ValidationThroughDepfile) { + const char* manifest = + "build out: cat in |@ validate\n" + "build validate: cat in2 | out\n" + "build out2: cat in3\n" + " deps = gcc\n" + " depfile = out2.d\n"; + + string err; + + { + fs_.Create("in", ""); + fs_.Create("in2", ""); + fs_.Create("in3", ""); + fs_.Create("out2.d", "out: out"); + + State state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + + EXPECT_TRUE(builder.AddTarget("out2", &err)); + ASSERT_EQ("", err); + + EXPECT_EQ(builder.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + + // On the first build, only the out2 command is run. + ASSERT_EQ(command_runner_.commands_ran_.size(), size_t(1)); + EXPECT_EQ("cat in3 > out2", command_runner_.commands_ran_[0]); + + // The deps file should have been removed. + EXPECT_EQ(0, fs_.Stat("out2.d", &err)); + + deps_log.Close(); + builder.command_runner_.release(); + } + + fs_.Tick(); + command_runner_.commands_ran_.clear(); + + { + fs_.Create("in2", ""); + fs_.Create("in3", ""); + + State state; + ASSERT_NO_FATAL_FAILURE(AddCatRule(&state)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, manifest)); + + DepsLog deps_log; + ASSERT_TRUE(deps_log.Load(deps_log_file_.path(), &state, &err)); + ASSERT_TRUE(deps_log.OpenForWrite(deps_log_file_.path(), &err)); + ASSERT_EQ("", err); + + Builder builder(&state, config_, NULL, &deps_log, &fs_, &status_, 0); + builder.command_runner_.reset(&command_runner_); + + EXPECT_TRUE(builder.AddTarget("out2", &err)); + ASSERT_EQ("", err); + + EXPECT_EQ(builder.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + + // The out and validate actions should have been run as well as out2. + ASSERT_EQ(command_runner_.commands_ran_.size(), size_t(3)); + // out has to run first, as both out2 and validate depend on it. + EXPECT_EQ("cat in > out", command_runner_.commands_ran_[0]); + + deps_log.Close(); + builder.command_runner_.release(); + } +} + +TEST_F(BuildTest, ValidationCircular) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out: cat in |@ out2\n" + "build out2: cat in2 |@ out\n")); + + fs_.Create("in", ""); + fs_.Create("in2", ""); + + string err; + EXPECT_TRUE(builder_.AddTarget("out", &err)); + EXPECT_EQ("", err); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + + EXPECT_EQ(2u, command_runner_.commands_ran_.size()); + + // Test touching "in" rebuilds "out". + fs_.Tick(); + fs_.Create("in", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in > out", command_runner_.commands_ran_[0]); + + // Test touching "in2" rebuilds "out2". + fs_.Tick(); + fs_.Create("in2", ""); + + err.clear(); + command_runner_.commands_ran_.clear(); + state_.Reset(); + EXPECT_TRUE(builder_.AddTarget("out", &err)); + ASSERT_EQ("", err); + + EXPECT_EQ(builder_.Build(&err), ExitSuccess); + EXPECT_EQ("", err); + + ASSERT_EQ(1u, command_runner_.commands_ran_.size()); + EXPECT_EQ("cat in2 > out2", command_runner_.commands_ran_[0]); +} + +TEST_F(BuildTest, ValidationWithCircularDependency) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out: cat in |@ validate\n" + "build validate: cat validate_in | out\n" + "build validate_in: cat validate\n")); + + fs_.Create("in", ""); + + string err; + EXPECT_FALSE(builder_.AddTarget("out", &err)); + EXPECT_EQ("dependency cycle: validate -> validate_in -> validate", err); +} diff --git a/src/canon_perftest.cc b/src/canon_perftest.cc index 03f4a2f1f9..6b5e3822d8 100644 --- a/src/canon_perftest.cc +++ b/src/canon_perftest.cc @@ -18,13 +18,14 @@ #include "util.h" #include "metrics.h" +using namespace std; + const char kPath[] = "../../third_party/WebKit/Source/WebCore/" "platform/leveldb/LevelDBWriteBatch.cpp"; int main() { vector times; - string err; char buf[200]; size_t len = strlen(kPath); @@ -35,7 +36,7 @@ int main() { int64_t start = GetTimeMillis(); uint64_t slash_bits; for (int i = 0; i < kNumRepetitions; ++i) { - CanonicalizePath(buf, &len, &slash_bits, &err); + CanonicalizePath(buf, &len, &slash_bits); } int delta = (int)(GetTimeMillis() - start); times.push_back(delta); diff --git a/src/clean.cc b/src/clean.cc index d1f221d916..ceffe64027 100644 --- a/src/clean.cc +++ b/src/clean.cc @@ -22,14 +22,14 @@ #include "state.h" #include "util.h" +using namespace std; + Cleaner::Cleaner(State* state, const BuildConfig& config, DiskInterface* disk_interface) : state_(state), config_(config), dyndep_loader_(state, disk_interface), - removed_(), - cleaned_(), cleaned_files_count_(0), disk_interface_(disk_interface), status_(0) { @@ -124,6 +124,29 @@ int Cleaner::CleanAll(bool generator) { return status_; } +int Cleaner::CleanDead(const BuildLog::Entries& entries) { + Reset(); + PrintHeader(); + LoadDyndeps(); + for (BuildLog::Entries::const_iterator i = entries.begin(); i != entries.end(); ++i) { + Node* n = state_->LookupNode(i->first); + // Detecting stale outputs works as follows: + // + // - If it has no Node, it is not in the build graph, or the deps log + // anymore, hence is stale. + // + // - If it isn't an output or input for any edge, it comes from a stale + // entry in the deps log, but no longer referenced from the build + // graph. + // + if (!n || (!n->in_edge() && n->out_edges().empty())) { + Remove(i->first.AsString()); + } + } + PrintFooter(); + return status_; +} + void Cleaner::DoCleanTarget(Node* target) { if (Edge* e = target->in_edge()) { // Do not try to remove phony targets @@ -176,21 +199,21 @@ int Cleaner::CleanTargets(int target_count, char* targets[]) { LoadDyndeps(); for (int i = 0; i < target_count; ++i) { string target_name = targets[i]; - uint64_t slash_bits; - string err; - if (!CanonicalizePath(&target_name, &slash_bits, &err)) { - Error("failed to canonicalize '%s': %s", target_name.c_str(), err.c_str()); + if (target_name.empty()) { + Error("failed to canonicalize '': empty path"); status_ = 1; + continue; + } + uint64_t slash_bits; + CanonicalizePath(&target_name, &slash_bits); + Node* target = state_->LookupNode(target_name); + if (target) { + if (IsVerbose()) + printf("Target %s\n", target_name.c_str()); + DoCleanTarget(target); } else { - Node* target = state_->LookupNode(target_name); - if (target) { - if (IsVerbose()) - printf("Target %s\n", target_name.c_str()); - DoCleanTarget(target); - } else { - Error("unknown target '%s'", target_name.c_str()); - status_ = 1; - } + Error("unknown target '%s'", target_name.c_str()); + status_ = 1; } } PrintFooter(); @@ -270,7 +293,8 @@ void Cleaner::LoadDyndeps() { // Load dyndep files that exist, before they are cleaned. for (vector::iterator e = state_->edges_.begin(); e != state_->edges_.end(); ++e) { - if (Node* dyndep = (*e)->dyndep_) { + Node* dyndep; + if ((dyndep = (*e)->dyndep_) && dyndep->dyndep_pending()) { // Capture and ignore errors loading the dyndep file. // We clean as much of the graph as we know. std::string err; diff --git a/src/clean.h b/src/clean.h index d044fb1200..cf3f1c3f3d 100644 --- a/src/clean.h +++ b/src/clean.h @@ -20,8 +20,7 @@ #include "build.h" #include "dyndep.h" - -using namespace std; +#include "build_log.h" struct State; struct Node; @@ -58,6 +57,10 @@ struct Cleaner { /// Clean the file produced by the given @a rules. /// @return non-zero if an error occurs. int CleanRules(int rule_count, char* rules[]); + /// Clean the files produced by previous builds that are no longer in the + /// manifest. + /// @return non-zero if an error occurs. + int CleanDead(const BuildLog::Entries& entries); /// @return the number of file cleaned. int cleaned_files_count() const { @@ -73,15 +76,15 @@ struct Cleaner { private: /// Remove the file @a path. /// @return whether the file has been removed. - int RemoveFile(const string& path); + int RemoveFile(const std::string& path); /// @returns whether the file @a path exists. - bool FileExists(const string& path); - void Report(const string& path); + bool FileExists(const std::string& path); + void Report(const std::string& path); /// Remove the given @a path file only if it has not been already removed. - void Remove(const string& path); + void Remove(const std::string& path); /// @return whether the given @a path has already been removed. - bool IsAlreadyRemoved(const string& path); + bool IsAlreadyRemoved(const std::string& path); /// Remove the depfile and rspfile for an Edge. void RemoveEdgeFiles(Edge* edge); @@ -98,8 +101,8 @@ struct Cleaner { State* state_; const BuildConfig& config_; DyndepLoader dyndep_loader_; - set removed_; - set cleaned_; + std::set removed_; + std::set cleaned_; int cleaned_files_count_; DiskInterface* disk_interface_; int status_; diff --git a/src/clean_test.cc b/src/clean_test.cc index 45187f4fa4..39aede37a1 100644 --- a/src/clean_test.cc +++ b/src/clean_test.cc @@ -15,8 +15,19 @@ #include "clean.h" #include "build.h" +#include "util.h" #include "test.h" +#ifndef _WIN32 +#include +#endif + +using namespace std; + +namespace { + +const char kTestFilename[] = "CleanTest-tempfile"; + struct CleanTest : public StateTestWithBuiltinRules { VirtualFileSystem fs_; BuildConfig config_; @@ -454,3 +465,137 @@ TEST_F(CleanTest, CleanDepFileAndRspFileWithSpaces) { EXPECT_EQ(0, fs_.Stat("out 1.d", &err)); EXPECT_EQ(0, fs_.Stat("out 2.rsp", &err)); } + +struct CleanDeadTest : public CleanTest, public BuildLogUser{ + virtual void SetUp() { + // In case a crashing test left a stale file behind. + platformAwareUnlink(kTestFilename); + CleanTest::SetUp(); + } + virtual void TearDown() { + platformAwareUnlink(kTestFilename); + } + virtual bool IsPathDead(StringPiece) const { return false; } +}; + +TEST_F(CleanDeadTest, CleanDead) { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, +"rule cat\n" +" command = cat $in > $out\n" +"build out1: cat in\n" +"build out2: cat in\n" +)); + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build out2: cat in\n" +)); + fs_.Create("in", ""); + fs_.Create("out1", ""); + fs_.Create("out2", ""); + + BuildLog log1; + string err; + EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err)); + ASSERT_EQ("", err); + log1.RecordCommand(state.edges_[0], 15, 18); + log1.RecordCommand(state.edges_[1], 20, 25); + log1.Close(); + + BuildLog log2; + EXPECT_TRUE(log2.Load(kTestFilename, &err)); + ASSERT_EQ("", err); + ASSERT_EQ(2u, log2.entries().size()); + ASSERT_TRUE(log2.LookupByOutput("out1")); + ASSERT_TRUE(log2.LookupByOutput("out2")); + + // First use the manifest that describe how to build out1. + Cleaner cleaner1(&state, config_, &fs_); + EXPECT_EQ(0, cleaner1.CleanDead(log2.entries())); + EXPECT_EQ(0, cleaner1.cleaned_files_count()); + EXPECT_EQ(0u, fs_.files_removed_.size()); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_NE(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + + // Then use the manifest that does not build out1 anymore. + Cleaner cleaner2(&state_, config_, &fs_); + EXPECT_EQ(0, cleaner2.CleanDead(log2.entries())); + EXPECT_EQ(1, cleaner2.cleaned_files_count()); + EXPECT_EQ(1u, fs_.files_removed_.size()); + EXPECT_EQ("out1", *(fs_.files_removed_.begin())); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_EQ(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + + // Nothing to do now. + EXPECT_EQ(0, cleaner2.CleanDead(log2.entries())); + EXPECT_EQ(0, cleaner2.cleaned_files_count()); + EXPECT_EQ(1u, fs_.files_removed_.size()); + EXPECT_EQ("out1", *(fs_.files_removed_.begin())); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_EQ(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + log2.Close(); +} + +TEST_F(CleanDeadTest, CleanDeadPreservesInputs) { + State state; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state, +"rule cat\n" +" command = cat $in > $out\n" +"build out1: cat in\n" +"build out2: cat in\n" +)); + // This manifest does not build out1 anymore, but makes + // it an implicit input. CleanDead should detect this + // and preserve it. + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build out2: cat in | out1\n" +)); + fs_.Create("in", ""); + fs_.Create("out1", ""); + fs_.Create("out2", ""); + + BuildLog log1; + string err; + EXPECT_TRUE(log1.OpenForWrite(kTestFilename, *this, &err)); + ASSERT_EQ("", err); + log1.RecordCommand(state.edges_[0], 15, 18); + log1.RecordCommand(state.edges_[1], 20, 25); + log1.Close(); + + BuildLog log2; + EXPECT_TRUE(log2.Load(kTestFilename, &err)); + ASSERT_EQ("", err); + ASSERT_EQ(2u, log2.entries().size()); + ASSERT_TRUE(log2.LookupByOutput("out1")); + ASSERT_TRUE(log2.LookupByOutput("out2")); + + // First use the manifest that describe how to build out1. + Cleaner cleaner1(&state, config_, &fs_); + EXPECT_EQ(0, cleaner1.CleanDead(log2.entries())); + EXPECT_EQ(0, cleaner1.cleaned_files_count()); + EXPECT_EQ(0u, fs_.files_removed_.size()); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_NE(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + + // Then use the manifest that does not build out1 anymore. + Cleaner cleaner2(&state_, config_, &fs_); + EXPECT_EQ(0, cleaner2.CleanDead(log2.entries())); + EXPECT_EQ(0, cleaner2.cleaned_files_count()); + EXPECT_EQ(0u, fs_.files_removed_.size()); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_NE(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + + // Nothing to do now. + EXPECT_EQ(0, cleaner2.CleanDead(log2.entries())); + EXPECT_EQ(0, cleaner2.cleaned_files_count()); + EXPECT_EQ(0u, fs_.files_removed_.size()); + EXPECT_NE(0, fs_.Stat("in", &err)); + EXPECT_NE(0, fs_.Stat("out1", &err)); + EXPECT_NE(0, fs_.Stat("out2", &err)); + log2.Close(); +} +} // anonymous namespace diff --git a/src/clparser.cc b/src/clparser.cc index 7994c06f4e..3d3e7de54e 100644 --- a/src/clparser.cc +++ b/src/clparser.cc @@ -28,6 +28,8 @@ #include "util.h" #endif +using namespace std; + namespace { /// Return true if \a input ends with \a needle. @@ -70,7 +72,8 @@ bool CLParser::FilterInputFilename(string line) { return EndsWith(line, ".c") || EndsWith(line, ".cc") || EndsWith(line, ".cxx") || - EndsWith(line, ".cpp"); + EndsWith(line, ".cpp") || + EndsWith(line, ".c++"); } // static @@ -81,6 +84,7 @@ bool CLParser::Parse(const string& output, const string& deps_prefix, // Loop over all lines in the output to process them. assert(&output != filtered_output); size_t start = 0; + bool seen_show_includes = false; #ifdef _WIN32 IncludesNormalize normalizer("."); #endif @@ -93,6 +97,7 @@ bool CLParser::Parse(const string& output, const string& deps_prefix, string include = FilterShowIncludes(line, deps_prefix); if (!include.empty()) { + seen_show_includes = true; string normalized; #ifdef _WIN32 if (!normalizer.Normalize(include, &normalized, err)) @@ -101,12 +106,11 @@ bool CLParser::Parse(const string& output, const string& deps_prefix, // TODO: should this make the path relative to cwd? normalized = include; uint64_t slash_bits; - if (!CanonicalizePath(&normalized, &slash_bits, err)) - return false; + CanonicalizePath(&normalized, &slash_bits); #endif if (!IsSystemInclude(normalized)) includes_.insert(normalized); - } else if (FilterInputFilename(line)) { + } else if (!seen_show_includes && FilterInputFilename(line)) { // Drop it. // TODO: if we support compiling multiple output files in a single // cl.exe invocation, we should stash the filename. diff --git a/src/clparser.h b/src/clparser.h index e597e7ebc2..2a33628ab0 100644 --- a/src/clparser.h +++ b/src/clparser.h @@ -17,7 +17,6 @@ #include #include -using namespace std; /// Visual Studio's cl.exe requires some massaging to work with Ninja; /// for example, it emits include information on stderr in a funny @@ -27,26 +26,26 @@ struct CLParser { /// Parse a line of cl.exe output and extract /showIncludes info. /// If a dependency is extracted, returns a nonempty string. /// Exposed for testing. - static string FilterShowIncludes(const string& line, - const string& deps_prefix); + static std::string FilterShowIncludes(const std::string& line, + const std::string& deps_prefix); /// Return true if a mentioned include file is a system path. /// Filtering these out reduces dependency information considerably. - static bool IsSystemInclude(string path); + static bool IsSystemInclude(std::string path); /// Parse a line of cl.exe output and return true if it looks like /// it's printing an input filename. This is a heuristic but it appears /// to be the best we can do. /// Exposed for testing. - static bool FilterInputFilename(string line); + static bool FilterInputFilename(std::string line); /// Parse the full output of cl, filling filtered_output with the text that /// should be printed (if any). Returns true on success, or false with err /// filled. output must not be the same object as filtered_object. - bool Parse(const string& output, const string& deps_prefix, - string* filtered_output, string* err); + bool Parse(const std::string& output, const std::string& deps_prefix, + std::string* filtered_output, std::string* err); - set includes_; + std::set includes_; }; #endif // NINJA_CLPARSER_H_ diff --git a/src/clparser_perftest.cc b/src/clparser_perftest.cc index 7ac52302b1..008ac465ad 100644 --- a/src/clparser_perftest.cc +++ b/src/clparser_perftest.cc @@ -18,6 +18,8 @@ #include "clparser.h" #include "metrics.h" +using namespace std; + int main(int argc, char* argv[]) { // Output of /showIncludes from #include string perf_testdata = diff --git a/src/clparser_test.cc b/src/clparser_test.cc index 1549ab1cb9..f1416809c3 100644 --- a/src/clparser_test.cc +++ b/src/clparser_test.cc @@ -17,6 +17,8 @@ #include "test.h" #include "util.h" +using namespace std; + TEST(CLParserTest, ShowIncludes) { ASSERT_EQ("", CLParser::FilterShowIncludes("", "")); @@ -68,6 +70,17 @@ TEST(CLParserTest, ParseFilenameFilter) { ASSERT_EQ("cl: warning\n", output); } +TEST(CLParserTest, NoFilenameFilterAfterShowIncludes) { + CLParser parser; + string output, err; + ASSERT_TRUE(parser.Parse( + "foo.cc\r\n" + "Note: including file: foo.h\r\n" + "something something foo.cc\r\n", + "", &output, &err)); + ASSERT_EQ("something something foo.cc\n", output); +} + TEST(CLParserTest, ParseSystemInclude) { CLParser parser; string output, err; diff --git a/src/command_collector.h b/src/command_collector.h new file mode 100644 index 0000000000..003af9fbb7 --- /dev/null +++ b/src/command_collector.h @@ -0,0 +1,65 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_COMMAND_COLLECTOR_H_ +#define NINJA_COMMAND_COLLECTOR_H_ + +#include +#include +#include + +#include "graph.h" + +/// Collects the transitive set of edges that lead into a given set +/// of starting nodes. Used to implement the `compdb-targets` tool. +/// +/// When collecting inputs, the outputs of phony edges are always ignored +/// from the result, but are followed by the dependency walk. +/// +/// Usage is: +/// - Create instance. +/// - Call CollectFrom() for each root node to collect edges from. +/// - Call TakeResult() to retrieve the list of edges. +/// +struct CommandCollector { + void CollectFrom(const Node* node) { + assert(node); + + if (!visited_nodes_.insert(node).second) + return; + + Edge* edge = node->in_edge(); + if (!edge || !visited_edges_.insert(edge).second) + return; + + for (Node* input_node : edge->inputs_) + CollectFrom(input_node); + + if (!edge->is_phony()) + in_edges.push_back(edge); + } + + private: + std::unordered_set visited_nodes_; + std::unordered_set visited_edges_; + + /// we use a vector to preserve order from requisites to their dependents. + /// This may help LSP server performance in languages that support modules, + /// but it also ensures that the output of `-t compdb-targets foo` is + /// consistent, which is useful in regression tests. + public: + std::vector in_edges; +}; + +#endif // NINJA_COMMAND_COLLECTOR_H_ diff --git a/src/debug_flags.cc b/src/debug_flags.cc index 44b14c483b..c83abb4093 100644 --- a/src/debug_flags.cc +++ b/src/debug_flags.cc @@ -12,6 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include +#include +#include + +#include "graph.h" + bool g_explaining = false; bool g_keep_depfile = false; diff --git a/src/debug_flags.h b/src/debug_flags.h index e08a43b438..fe73a5202c 100644 --- a/src/debug_flags.h +++ b/src/debug_flags.h @@ -17,10 +17,8 @@ #include -#define EXPLAIN(fmt, ...) { \ - if (g_explaining) \ - fprintf(stderr, "ninja explain: " fmt "\n", __VA_ARGS__); \ -} +struct Edge; +struct Node; extern bool g_explaining; diff --git a/src/depfile_parser.cc b/src/depfile_parser.cc index 6faeac61b4..9f7b393b4b 100644 --- a/src/depfile_parser.cc +++ b/src/depfile_parser.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 1.1.1 */ +/* Generated by re2c */ // Copyright 2011 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +16,10 @@ #include "depfile_parser.h" #include "util.h" +#include + +using namespace std; + DepfileParser::DepfileParser(DepfileParserOptions options) : options_(options) { @@ -48,10 +52,9 @@ bool DepfileParser::Parse(string* content, string* err) { char* in = &(*content)[0]; char* end = in + content->size(); bool have_target = false; - bool have_secondary_target_on_this_rule = false; - bool have_newline_since_primary_target = false; - bool warned_distinct_target_lines = false; bool parsing_targets = true; + bool poisoned_input = false; + bool is_empty = true; while (in < end) { bool have_newline = false; // out: current output point (typically same as in, but can fall behind @@ -74,7 +77,7 @@ bool DepfileParser::Parse(string* content, string* err) { 0, 128, 0, 0, 0, 128, 0, 0, 128, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, - 128, 128, 128, 0, 0, 128, 0, 0, + 128, 128, 128, 0, 0, 128, 0, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, @@ -102,55 +105,55 @@ bool DepfileParser::Parse(string* content, string* err) { }; yych = *in; if (yybm[0+yych] & 128) { - goto yy9; + goto yy5; } if (yych <= '\r') { if (yych <= '\t') { - if (yych >= 0x01) goto yy4; + if (yych >= 0x01) goto yy1; } else { - if (yych <= '\n') goto yy6; - if (yych <= '\f') goto yy4; - goto yy8; + if (yych <= '\n') goto yy3; + if (yych <= '\f') goto yy1; + goto yy4; } } else { if (yych <= '$') { - if (yych <= '#') goto yy4; - goto yy12; + if (yych <= '#') goto yy1; + goto yy7; } else { - if (yych <= '?') goto yy4; - if (yych <= '\\') goto yy13; - goto yy4; + if (yych <= '>') goto yy1; + if (yych <= '\\') goto yy8; + goto yy1; } } ++in; { break; } -yy4: +yy1: ++in; -yy5: +yy2: { // For any other character (e.g. whitespace), swallow it here, // allowing the outer logic to loop around again. break; } -yy6: +yy3: ++in; { // A newline ends the current file name and the current rule. have_newline = true; break; } -yy8: +yy4: yych = *++in; - if (yych == '\n') goto yy6; - goto yy5; -yy9: + if (yych == '\n') goto yy3; + goto yy2; +yy5: yych = *++in; if (yybm[0+yych] & 128) { - goto yy9; + goto yy5; } -yy11: +yy6: { // Got a span of plain text. int len = (int)(in - start); @@ -160,53 +163,54 @@ bool DepfileParser::Parse(string* content, string* err) { out += len; continue; } -yy12: +yy7: yych = *++in; - if (yych == '$') goto yy14; - goto yy5; -yy13: + if (yych == '$') goto yy9; + goto yy2; +yy8: yych = *(yymarker = ++in); - if (yych <= 0x1F) { + if (yych <= ' ') { if (yych <= '\n') { - if (yych <= 0x00) goto yy5; - if (yych <= '\t') goto yy16; - goto yy17; + if (yych <= 0x00) goto yy2; + if (yych <= '\t') goto yy10; + goto yy11; } else { - if (yych == '\r') goto yy19; - goto yy16; + if (yych == '\r') goto yy12; + if (yych <= 0x1F) goto yy10; + goto yy13; } } else { - if (yych <= '#') { - if (yych <= ' ') goto yy21; - if (yych <= '"') goto yy16; - goto yy23; + if (yych <= '9') { + if (yych == '#') goto yy14; + goto yy10; } else { - if (yych == '\\') goto yy25; - goto yy16; + if (yych <= ':') goto yy15; + if (yych == '\\') goto yy17; + goto yy10; } } -yy14: +yy9: ++in; { // De-escape dollar character. *out++ = '$'; continue; } -yy16: +yy10: ++in; - goto yy11; -yy17: + goto yy6; +yy11: ++in; { // A line continuation ends the current file name. break; } -yy19: +yy12: yych = *++in; - if (yych == '\n') goto yy17; + if (yych == '\n') goto yy11; in = yymarker; - goto yy5; -yy21: + goto yy2; +yy13: ++in; { // 2N+1 backslashes plus space -> N backslashes plus space. @@ -218,7 +222,7 @@ bool DepfileParser::Parse(string* content, string* err) { *out++ = ' '; continue; } -yy23: +yy14: ++in; { // De-escape hash sign, but preserve other leading backslashes. @@ -229,28 +233,65 @@ bool DepfileParser::Parse(string* content, string* err) { *out++ = '#'; continue; } -yy25: +yy15: yych = *++in; - if (yych <= 0x1F) { + if (yych <= '\f') { + if (yych <= 0x00) goto yy18; + if (yych <= 0x08) goto yy16; + if (yych <= '\n') goto yy18; + } else { + if (yych <= '\r') goto yy18; + if (yych == ' ') goto yy18; + } +yy16: + { + // De-escape colon sign, but preserve other leading backslashes. + // Regular expression uses lookahead to make sure that no whitespace + // nor EOF follows. In that case it'd be the : at the end of a target + int len = (int)(in - start); + if (len > 2 && out < start) + memset(out, '\\', len - 2); + out += len - 2; + *out++ = ':'; + continue; + } +yy17: + yych = *++in; + if (yych <= ' ') { if (yych <= '\n') { - if (yych <= 0x00) goto yy11; - if (yych <= '\t') goto yy16; - goto yy11; + if (yych <= 0x00) goto yy6; + if (yych <= '\t') goto yy10; + goto yy6; } else { - if (yych == '\r') goto yy11; - goto yy16; + if (yych == '\r') goto yy6; + if (yych <= 0x1F) goto yy10; + goto yy19; } } else { - if (yych <= '#') { - if (yych <= ' ') goto yy26; - if (yych <= '"') goto yy16; - goto yy23; + if (yych <= '9') { + if (yych == '#') goto yy14; + goto yy10; } else { - if (yych == '\\') goto yy28; - goto yy16; + if (yych <= ':') goto yy15; + if (yych == '\\') goto yy20; + goto yy10; } } -yy26: +yy18: + ++in; + { + // Backslash followed by : and whitespace. + // It is therefore normal text and not an escaped colon + int len = (int)(in - start - 1); + // Need to shift it over if we're overwriting backslashes. + if (out < start) + memmove(out, start, len); + out += len; + if (*(in - 1) == '\n') + have_newline = true; + break; + } +yy19: ++in; { // 2N backslashes plus space -> 2N backslashes, end of filename. @@ -260,25 +301,26 @@ bool DepfileParser::Parse(string* content, string* err) { out += len - 1; break; } -yy28: +yy20: yych = *++in; - if (yych <= 0x1F) { + if (yych <= ' ') { if (yych <= '\n') { - if (yych <= 0x00) goto yy11; - if (yych <= '\t') goto yy16; - goto yy11; + if (yych <= 0x00) goto yy6; + if (yych <= '\t') goto yy10; + goto yy6; } else { - if (yych == '\r') goto yy11; - goto yy16; + if (yych == '\r') goto yy6; + if (yych <= 0x1F) goto yy10; + goto yy13; } } else { - if (yych <= '#') { - if (yych <= ' ') goto yy21; - if (yych <= '"') goto yy16; - goto yy23; + if (yych <= '9') { + if (yych == '#') goto yy14; + goto yy10; } else { - if (yych == '\\') goto yy25; - goto yy16; + if (yych <= ':') goto yy15; + if (yych == '\\') goto yy17; + goto yy10; } } } @@ -294,44 +336,36 @@ bool DepfileParser::Parse(string* content, string* err) { } if (len > 0) { - if (is_dependency) { - if (have_secondary_target_on_this_rule) { - if (!have_newline_since_primary_target) { - *err = "depfile has multiple output paths"; + is_empty = false; + StringPiece piece = StringPiece(filename, len); + // If we've seen this as an input before, skip it. + std::vector::iterator pos = std::find(ins_.begin(), ins_.end(), piece); + if (pos == ins_.end()) { + if (is_dependency) { + if (poisoned_input) { + *err = "inputs may not also have inputs"; return false; - } else if (options_.depfile_distinct_target_lines_action_ == - kDepfileDistinctTargetLinesActionError) { - *err = - "depfile has multiple output paths (on separate lines)" - " [-w depfilemulti=err]"; - return false; - } else { - if (!warned_distinct_target_lines) { - warned_distinct_target_lines = true; - Warning("depfile has multiple output paths (on separate lines); " - "continuing anyway [-w depfilemulti=warn]"); - } - continue; } + // New input. + ins_.push_back(piece); + } else { + // Check for a new output. + if (std::find(outs_.begin(), outs_.end(), piece) == outs_.end()) + outs_.push_back(piece); } - ins_.push_back(StringPiece(filename, len)); - } else if (!out_.str_) { - out_ = StringPiece(filename, len); - } else if (out_ != StringPiece(filename, len)) { - have_secondary_target_on_this_rule = true; + } else if (!is_dependency) { + // We've passed an input on the left side; reject new inputs. + poisoned_input = true; } } if (have_newline) { // A newline ends a rule so the next filename will be a new target. parsing_targets = true; - have_secondary_target_on_this_rule = false; - if (have_target) { - have_newline_since_primary_target = true; - } + poisoned_input = false; } } - if (!have_target) { + if (!have_target && !is_empty) { *err = "expected ':' in depfile"; return false; } diff --git a/src/depfile_parser.h b/src/depfile_parser.h index be203746d6..0e8db81a6b 100644 --- a/src/depfile_parser.h +++ b/src/depfile_parser.h @@ -17,21 +17,11 @@ #include #include -using namespace std; #include "string_piece.h" -enum DepfileDistinctTargetLinesAction { - kDepfileDistinctTargetLinesActionWarn, - kDepfileDistinctTargetLinesActionError, -}; - struct DepfileParserOptions { - DepfileParserOptions() - : depfile_distinct_target_lines_action_( - kDepfileDistinctTargetLinesActionWarn) {} - DepfileDistinctTargetLinesAction - depfile_distinct_target_lines_action_; + DepfileParserOptions() {} }; /// Parser for the dependency information emitted by gcc's -M flags. @@ -42,10 +32,10 @@ struct DepfileParser { /// Parse an input file. Input must be NUL-terminated. /// Warning: may mutate the content in-place and parsed StringPieces are /// pointers within it. - bool Parse(string* content, string* err); + bool Parse(std::string* content, std::string* err); - StringPiece out_; - vector ins_; + std::vector outs_; + std::vector ins_; DepfileParserOptions options_; }; diff --git a/src/depfile_parser.in.cc b/src/depfile_parser.in.cc index 735a0c3a0e..5bef2585bf 100644 --- a/src/depfile_parser.in.cc +++ b/src/depfile_parser.in.cc @@ -15,6 +15,10 @@ #include "depfile_parser.h" #include "util.h" +#include + +using namespace std; + DepfileParser::DepfileParser(DepfileParserOptions options) : options_(options) { @@ -47,10 +51,9 @@ bool DepfileParser::Parse(string* content, string* err) { char* in = &(*content)[0]; char* end = in + content->size(); bool have_target = false; - bool have_secondary_target_on_this_rule = false; - bool have_newline_since_primary_target = false; - bool warned_distinct_target_lines = false; bool parsing_targets = true; + bool poisoned_input = false; + bool is_empty = true; while (in < end) { bool have_newline = false; // out: current output point (typically same as in, but can fall behind @@ -103,12 +106,35 @@ bool DepfileParser::Parse(string* content, string* err) { *out++ = '#'; continue; } + '\\'+ ':' [\x00\x20\r\n\t] { + // Backslash followed by : and whitespace. + // It is therefore normal text and not an escaped colon + int len = (int)(in - start - 1); + // Need to shift it over if we're overwriting backslashes. + if (out < start) + memmove(out, start, len); + out += len; + if (*(in - 1) == '\n') + have_newline = true; + break; + } + '\\'+ ':' { + // De-escape colon sign, but preserve other leading backslashes. + // Regular expression uses lookahead to make sure that no whitespace + // nor EOF follows. In that case it'd be the : at the end of a target + int len = (int)(in - start); + if (len > 2 && out < start) + memset(out, '\\', len - 2); + out += len - 2; + *out++ = ':'; + continue; + } '$$' { // De-escape dollar character. *out++ = '$'; continue; } - '\\'+ [^\000\r\n] | [a-zA-Z0-9+,/_:.~()}{%=@\x5B\x5D!\x80-\xFF-]+ { + '\\'+ [^\000\r\n] | [a-zA-Z0-9+?,/_:.~()}{%=@\x5B\x5D!\x80-\xFF-]+ { // Got a span of plain text. int len = (int)(in - start); // Need to shift it over if we're overwriting backslashes. @@ -146,44 +172,36 @@ bool DepfileParser::Parse(string* content, string* err) { } if (len > 0) { - if (is_dependency) { - if (have_secondary_target_on_this_rule) { - if (!have_newline_since_primary_target) { - *err = "depfile has multiple output paths"; + is_empty = false; + StringPiece piece = StringPiece(filename, len); + // If we've seen this as an input before, skip it. + std::vector::iterator pos = std::find(ins_.begin(), ins_.end(), piece); + if (pos == ins_.end()) { + if (is_dependency) { + if (poisoned_input) { + *err = "inputs may not also have inputs"; return false; - } else if (options_.depfile_distinct_target_lines_action_ == - kDepfileDistinctTargetLinesActionError) { - *err = - "depfile has multiple output paths (on separate lines)" - " [-w depfilemulti=err]"; - return false; - } else { - if (!warned_distinct_target_lines) { - warned_distinct_target_lines = true; - Warning("depfile has multiple output paths (on separate lines); " - "continuing anyway [-w depfilemulti=warn]"); - } - continue; } + // New input. + ins_.push_back(piece); + } else { + // Check for a new output. + if (std::find(outs_.begin(), outs_.end(), piece) == outs_.end()) + outs_.push_back(piece); } - ins_.push_back(StringPiece(filename, len)); - } else if (!out_.str_) { - out_ = StringPiece(filename, len); - } else if (out_ != StringPiece(filename, len)) { - have_secondary_target_on_this_rule = true; + } else if (!is_dependency) { + // We've passed an input on the left side; reject new inputs. + poisoned_input = true; } } if (have_newline) { // A newline ends a rule so the next filename will be a new target. parsing_targets = true; - have_secondary_target_on_this_rule = false; - if (have_target) { - have_newline_since_primary_target = true; - } + poisoned_input = false; } } - if (!have_target) { + if (!have_target && !is_empty) { *err = "expected ':' in depfile"; return false; } diff --git a/src/depfile_parser_perftest.cc b/src/depfile_parser_perftest.cc index b21522168d..52555e6b2c 100644 --- a/src/depfile_parser_perftest.cc +++ b/src/depfile_parser_perftest.cc @@ -19,6 +19,8 @@ #include "util.h" #include "metrics.h" +using namespace std; + int main(int argc, char* argv[]) { if (argc < 2) { printf("usage: %s \n", argv[0]); diff --git a/src/depfile_parser_test.cc b/src/depfile_parser_test.cc index 19224f35ba..4c379fdcf3 100644 --- a/src/depfile_parser_test.cc +++ b/src/depfile_parser_test.cc @@ -16,6 +16,8 @@ #include "test.h" +using namespace std; + struct DepfileParserTest : public testing::Test { bool Parse(const char* input, string* err); @@ -34,7 +36,8 @@ TEST_F(DepfileParserTest, Basic) { "build/ninja.o: ninja.cc ninja.h eval_env.h manifest_parser.h\n", &err)); ASSERT_EQ("", err); - EXPECT_EQ("build/ninja.o", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + EXPECT_EQ("build/ninja.o", parser_.outs_[0].AsString()); EXPECT_EQ(4u, parser_.ins_.size()); } @@ -54,10 +57,21 @@ TEST_F(DepfileParserTest, Continuation) { " bar.h baz.h\n", &err)); ASSERT_EQ("", err); - EXPECT_EQ("foo.o", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + EXPECT_EQ("foo.o", parser_.outs_[0].AsString()); EXPECT_EQ(2u, parser_.ins_.size()); } +TEST_F(DepfileParserTest, WindowsDrivePaths) { + string err; + EXPECT_TRUE(Parse("foo.o: //?/c:/bar.h\n", &err)); + ASSERT_EQ("", err); + ASSERT_EQ(1u, parser_.outs_.size()); + EXPECT_EQ("foo.o", parser_.outs_[0].AsString()); + EXPECT_EQ(1u, parser_.ins_.size()); + EXPECT_EQ("//?/c:/bar.h", parser_.ins_[0].AsString()); +} + TEST_F(DepfileParserTest, CarriageReturnContinuation) { string err; EXPECT_TRUE(Parse( @@ -65,7 +79,8 @@ TEST_F(DepfileParserTest, CarriageReturnContinuation) { " bar.h baz.h\r\n", &err)); ASSERT_EQ("", err); - EXPECT_EQ("foo.o", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + EXPECT_EQ("foo.o", parser_.outs_[0].AsString()); EXPECT_EQ(2u, parser_.ins_.size()); } @@ -79,8 +94,9 @@ TEST_F(DepfileParserTest, BackSlashes) { " Project\\Thing\\Bar.tlb \\\n", &err)); ASSERT_EQ("", err); + ASSERT_EQ(1u, parser_.outs_.size()); EXPECT_EQ("Project\\Dir\\Build\\Release8\\Foo\\Foo.res", - parser_.out_.AsString()); + parser_.outs_[0].AsString()); EXPECT_EQ(4u, parser_.ins_.size()); } @@ -90,8 +106,9 @@ TEST_F(DepfileParserTest, Spaces) { "a\\ bc\\ def: a\\ b c d", &err)); ASSERT_EQ("", err); + ASSERT_EQ(1u, parser_.outs_.size()); EXPECT_EQ("a bc def", - parser_.out_.AsString()); + parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("a b", parser_.ins_[0].AsString()); @@ -111,8 +128,9 @@ TEST_F(DepfileParserTest, MultipleBackslashes) { "a\\ b\\#c.h: \\\\\\\\\\ \\\\\\\\ \\\\share\\info\\\\#1", &err)); ASSERT_EQ("", err); + ASSERT_EQ(1u, parser_.outs_.size()); EXPECT_EQ("a b#c.h", - parser_.out_.AsString()); + parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("\\\\ ", parser_.ins_[0].AsString()); @@ -130,11 +148,47 @@ TEST_F(DepfileParserTest, Escapes) { "\\!\\@\\#$$\\%\\^\\&\\[\\]\\\\:", &err)); ASSERT_EQ("", err); + ASSERT_EQ(1u, parser_.outs_.size()); EXPECT_EQ("\\!\\@#$\\%\\^\\&\\[\\]\\\\", - parser_.out_.AsString()); + parser_.outs_[0].AsString()); ASSERT_EQ(0u, parser_.ins_.size()); } +TEST_F(DepfileParserTest, EscapedColons) +{ + std::string err; + // Tests for correct parsing of depfiles produced on Windows + // by both Clang, GCC pre 10 and GCC 10 + EXPECT_TRUE(Parse( +"c\\:\\gcc\\x86_64-w64-mingw32\\include\\stddef.o: \\\n" +" c:\\gcc\\x86_64-w64-mingw32\\include\\stddef.h \n", + &err)); + ASSERT_EQ("", err); + ASSERT_EQ(1u, parser_.outs_.size()); + EXPECT_EQ("c:\\gcc\\x86_64-w64-mingw32\\include\\stddef.o", + parser_.outs_[0].AsString()); + ASSERT_EQ(1u, parser_.ins_.size()); + EXPECT_EQ("c:\\gcc\\x86_64-w64-mingw32\\include\\stddef.h", + parser_.ins_[0].AsString()); +} + +TEST_F(DepfileParserTest, EscapedTargetColon) +{ + std::string err; + EXPECT_TRUE(Parse( +"foo1\\: x\n" +"foo1\\:\n" +"foo1\\:\r\n" +"foo1\\:\t\n" +"foo1\\:", + &err)); + ASSERT_EQ("", err); + ASSERT_EQ(1u, parser_.outs_.size()); + EXPECT_EQ("foo1\\", parser_.outs_[0].AsString()); + ASSERT_EQ(1u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); +} + TEST_F(DepfileParserTest, SpecialChars) { // See filenames like istreambuf.iterator_op!= in // https://github.com/google/libcxx/tree/master/test/iterators/stream.iterators/istreambuf.iterator/ @@ -147,8 +201,9 @@ TEST_F(DepfileParserTest, SpecialChars) { " a[1]b@2%c", &err)); ASSERT_EQ("", err); + ASSERT_EQ(1u, parser_.outs_.size()); EXPECT_EQ("C:/Program Files (x86)/Microsoft crtdefs.h", - parser_.out_.AsString()); + parser_.outs_[0].AsString()); ASSERT_EQ(5u, parser_.ins_.size()); EXPECT_EQ("en@quot.header~", parser_.ins_[0].AsString()); @@ -166,18 +221,25 @@ TEST_F(DepfileParserTest, UnifyMultipleOutputs) { // check that multiple duplicate targets are properly unified string err; EXPECT_TRUE(Parse("foo foo: x y z", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); EXPECT_EQ("z", parser_.ins_[2].AsString()); } -TEST_F(DepfileParserTest, RejectMultipleDifferentOutputs) { - // check that multiple different outputs are rejected by the parser +TEST_F(DepfileParserTest, MultipleDifferentOutputs) { + // check that multiple different outputs are accepted by the parser string err; - EXPECT_FALSE(Parse("foo bar: x y z", &err)); - ASSERT_EQ("depfile has multiple output paths", err); + EXPECT_TRUE(Parse("foo bar: x y z", &err)); + ASSERT_EQ(2u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); + ASSERT_EQ("bar", parser_.outs_[1].AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); } TEST_F(DepfileParserTest, MultipleEmptyRules) { @@ -185,7 +247,8 @@ TEST_F(DepfileParserTest, MultipleEmptyRules) { EXPECT_TRUE(Parse("foo: x\n" "foo: \n" "foo:\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(1u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); } @@ -196,7 +259,8 @@ TEST_F(DepfileParserTest, UnifyMultipleRulesLF) { "foo: y\n" "foo \\\n" "foo: z\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -209,7 +273,8 @@ TEST_F(DepfileParserTest, UnifyMultipleRulesCRLF) { "foo: y\r\n" "foo \\\r\n" "foo: z\r\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -222,7 +287,8 @@ TEST_F(DepfileParserTest, UnifyMixedRulesLF) { " y\n" "foo \\\n" "foo: z\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -235,7 +301,8 @@ TEST_F(DepfileParserTest, UnifyMixedRulesCRLF) { " y\r\n" "foo \\\r\n" "foo: z\r\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -247,7 +314,8 @@ TEST_F(DepfileParserTest, IndentedRulesLF) { EXPECT_TRUE(Parse(" foo: x\n" " foo: y\n" " foo: z\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -259,7 +327,8 @@ TEST_F(DepfileParserTest, IndentedRulesCRLF) { EXPECT_TRUE(Parse(" foo: x\r\n" " foo: y\r\n" " foo: z\r\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -272,7 +341,8 @@ TEST_F(DepfileParserTest, TolerateMP) { "x:\n" "y:\n" "z:\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); @@ -287,25 +357,55 @@ TEST_F(DepfileParserTest, MultipleRulesTolerateMP) { "y:\n" "foo: z\n" "z:\n", &err)); - ASSERT_EQ("foo", parser_.out_.AsString()); + ASSERT_EQ(1u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); ASSERT_EQ(3u, parser_.ins_.size()); EXPECT_EQ("x", parser_.ins_[0].AsString()); EXPECT_EQ("y", parser_.ins_[1].AsString()); EXPECT_EQ("z", parser_.ins_[2].AsString()); } -TEST_F(DepfileParserTest, MultipleRulesRejectDifferentOutputs) { - // check that multiple different outputs are rejected by the parser +TEST_F(DepfileParserTest, MultipleRulesDifferentOutputs) { + // check that multiple different outputs are accepted by the parser // when spread across multiple rules - DepfileParserOptions parser_opts; - parser_opts.depfile_distinct_target_lines_action_ = - kDepfileDistinctTargetLinesActionError; - DepfileParser parser(parser_opts); string err; - string input = - "foo: x y\n" - "bar: y z\n"; - EXPECT_FALSE(parser.Parse(&input, &err)); - ASSERT_EQ("depfile has multiple output paths (on separate lines)" - " [-w depfilemulti=err]", err); + EXPECT_TRUE(Parse("foo: x y\n" + "bar: y z\n", &err)); + ASSERT_EQ(2u, parser_.outs_.size()); + ASSERT_EQ("foo", parser_.outs_[0].AsString()); + ASSERT_EQ("bar", parser_.outs_[1].AsString()); + ASSERT_EQ(3u, parser_.ins_.size()); + EXPECT_EQ("x", parser_.ins_[0].AsString()); + EXPECT_EQ("y", parser_.ins_[1].AsString()); + EXPECT_EQ("z", parser_.ins_[2].AsString()); +} + +TEST_F(DepfileParserTest, BuggyMP) { + std::string err; + EXPECT_FALSE(Parse("foo: x y z\n" + "x: alsoin\n" + "y:\n" + "z:\n", &err)); + ASSERT_EQ("inputs may not also have inputs", err); +} + +TEST_F(DepfileParserTest, EmptyFile) { + std::string err; + EXPECT_TRUE(Parse("", &err)); + ASSERT_EQ(0u, parser_.outs_.size()); + ASSERT_EQ(0u, parser_.ins_.size()); +} + +TEST_F(DepfileParserTest, EmptyLines) { + std::string err; + EXPECT_TRUE(Parse("\n\n", &err)); + ASSERT_EQ(0u, parser_.outs_.size()); + ASSERT_EQ(0u, parser_.ins_.size()); +} + +TEST_F(DepfileParserTest, MissingColon) { + // The file is not empty but is missing a colon separator. + std::string err; + EXPECT_FALSE(Parse("foo.o foo.c\n", &err)); + EXPECT_EQ("expected ':' in depfile", err); } diff --git a/src/deps_log.cc b/src/deps_log.cc index 4aaffeb44b..fa59ef20cc 100644 --- a/src/deps_log.cc +++ b/src/deps_log.cc @@ -15,8 +15,9 @@ #include "deps_log.h" #include -#include #include +#include +#include #include #ifndef _WIN32 #include @@ -30,14 +31,18 @@ typedef unsigned __int32 uint32_t; #include "state.h" #include "util.h" +using namespace std; + // The version is stored as 4 bytes after the signature and also serves as a // byte order mark. Signature and version combined are 16 bytes long. -const char kFileSignature[] = "# ninjadeps\n"; -const int kCurrentVersion = 4; +static const char kFileSignature[] = "# ninjadeps\n"; +static const size_t kFileSignatureSize = sizeof(kFileSignature) - 1u; + +static const int32_t kCurrentVersion = 4; // Record size is currently limited to less than the full 32 bit, due to // internal buffers having to have this size. -const unsigned kMaxRecordSize = (1 << 19) - 1; +static constexpr size_t kMaxRecordSize = (1 << 19) - 1; DepsLog::~DepsLog() { Close(); @@ -49,45 +54,19 @@ bool DepsLog::OpenForWrite(const string& path, string* err) { return false; } - file_ = fopen(path.c_str(), "ab"); - if (!file_) { - *err = strerror(errno); - return false; - } - // Set the buffer size to this and flush the file buffer after every record - // to make sure records aren't written partially. - setvbuf(file_, NULL, _IOFBF, kMaxRecordSize + 1); - SetCloseOnExec(fileno(file_)); - - // Opening a file in append mode doesn't set the file pointer to the file's - // end on Windows. Do that explicitly. - fseek(file_, 0, SEEK_END); - - if (ftell(file_) == 0) { - if (fwrite(kFileSignature, sizeof(kFileSignature) - 1, 1, file_) < 1) { - *err = strerror(errno); - return false; - } - if (fwrite(&kCurrentVersion, 4, 1, file_) < 1) { - *err = strerror(errno); - return false; - } - } - if (fflush(file_) != 0) { - *err = strerror(errno); - return false; - } + assert(!file_); + file_path_ = path; // we don't actually open the file right now, but will do + // so on the first write attempt return true; } bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, const vector& nodes) { - return RecordDeps(node, mtime, nodes.size(), - nodes.empty() ? NULL : (Node**)&nodes.front()); + return RecordDeps(node, mtime, nodes.size(), nodes.data()); } -bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, - int node_count, Node** nodes) { +bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, int node_count, + Node* const* nodes) { // Track whether there's any new data to be recorded. bool made_change = false; @@ -132,6 +111,10 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, errno = ERANGE; return false; } + + if (!OpenForWriteIfNeeded()) { + return false; + } size |= 0x80000000; // Deps record: set high bit. if (fwrite(&size, 4, 1, file_) < 1) return false; @@ -162,52 +145,53 @@ bool DepsLog::RecordDeps(Node* node, TimeStamp mtime, } void DepsLog::Close() { + OpenForWriteIfNeeded(); // create the file even if nothing has been recorded if (file_) fclose(file_); file_ = NULL; } -bool DepsLog::Load(const string& path, State* state, string* err) { +LoadStatus DepsLog::Load(const string& path, State* state, string* err) { METRIC_RECORD(".ninja_deps load"); char buf[kMaxRecordSize + 1]; FILE* f = fopen(path.c_str(), "rb"); if (!f) { if (errno == ENOENT) - return true; + return LOAD_NOT_FOUND; *err = strerror(errno); - return false; + return LOAD_ERROR; } - bool valid_header = true; - int version = 0; - if (!fgets(buf, sizeof(buf), f) || fread(&version, 4, 1, f) < 1) - valid_header = false; + bool valid_header = fread(buf, kFileSignatureSize, 1, f) == 1 && + !memcmp(buf, kFileSignature, kFileSignatureSize); + + int32_t version = 0; + bool valid_version = + fread(&version, 4, 1, f) == 1 && version == kCurrentVersion; + // Note: For version differences, this should migrate to the new format. // But the v1 format could sometimes (rarely) end up with invalid data, so // don't migrate v1 to v3 to force a rebuild. (v2 only existed for a few days, // and there was no release with it, so pretend that it never happened.) - if (!valid_header || strcmp(buf, kFileSignature) != 0 || - version != kCurrentVersion) { + if (!valid_header || !valid_version) { if (version == 1) *err = "deps log version change; rebuilding"; else *err = "bad deps log signature or version; starting over"; fclose(f); - unlink(path.c_str()); + platformAwareUnlink(path.c_str()); // Don't report this as a failure. An empty deps log will cause // us to rebuild the outputs anyway. - return true; + return LOAD_SUCCESS; } - long offset; + long offset = ftell(f); bool read_failed = false; int unique_dep_record_count = 0; int total_dep_record_count = 0; for (;;) { - offset = ftell(f); - unsigned size; - if (fread(&size, 4, 1, f) < 1) { + if (fread(&size, sizeof(size), 1, f) < 1) { if (!feof(f)) read_failed = true; break; @@ -219,9 +203,13 @@ bool DepsLog::Load(const string& path, State* state, string* err) { read_failed = true; break; } + offset += size + sizeof(size); if (is_deps) { - assert(size % 4 == 0); + if ((size % 4) != 0) { + read_failed = true; + break; + } int* deps_data = reinterpret_cast(buf); int out_id = deps_data[0]; TimeStamp mtime; @@ -230,10 +218,18 @@ bool DepsLog::Load(const string& path, State* state, string* err) { deps_data += 3; int deps_count = (size / 4) - 3; + for (int i = 0; i < deps_count; ++i) { + int node_id = deps_data[i]; + if (node_id >= (int)nodes_.size() || !nodes_[node_id]) { + read_failed = true; + break; + } + } + if (read_failed) + break; + Deps* deps = new Deps(mtime, deps_count); for (int i = 0; i < deps_count; ++i) { - assert(deps_data[i] < (int)nodes_.size()); - assert(nodes_[deps_data[i]]); deps->nodes[i] = nodes_[deps_data[i]]; } @@ -242,7 +238,10 @@ bool DepsLog::Load(const string& path, State* state, string* err) { ++unique_dep_record_count; } else { int path_size = size - 4; - assert(path_size > 0); // CanonicalizePath() rejects empty paths. + if (path_size <= 0) { + read_failed = true; + break; + } // There can be up to 3 bytes of padding. if (buf[path_size - 1] == '\0') --path_size; if (buf[path_size - 1] == '\0') --path_size; @@ -262,12 +261,10 @@ bool DepsLog::Load(const string& path, State* state, string* err) { unsigned checksum = *reinterpret_cast(buf + size - 4); int expected_id = ~checksum; int id = nodes_.size(); - if (id != expected_id) { + if (id != expected_id || node->id() >= 0) { read_failed = true; break; } - - assert(node->id() < 0); node->set_id(id); nodes_.push_back(node); } @@ -284,12 +281,12 @@ bool DepsLog::Load(const string& path, State* state, string* err) { fclose(f); if (!Truncate(path, offset, err)) - return false; + return LOAD_ERROR; // The truncate succeeded; we'll just report the load error as a // warning because the build can proceed. *err += "; recovering"; - return true; + return LOAD_SUCCESS; } fclose(f); @@ -302,7 +299,7 @@ bool DepsLog::Load(const string& path, State* state, string* err) { needs_recompaction_ = true; } - return true; + return LOAD_SUCCESS; } DepsLog::Deps* DepsLog::GetDeps(Node* node) { @@ -313,6 +310,19 @@ DepsLog::Deps* DepsLog::GetDeps(Node* node) { return deps_[node->id()]; } +Node* DepsLog::GetFirstReverseDepsNode(Node* node) { + for (size_t id = 0; id < deps_.size(); ++id) { + Deps* deps = deps_[id]; + if (!deps) + continue; + for (int i = 0; i < deps->node_count; ++i) { + if (deps->nodes[i] == node) + return nodes_[id]; + } + } + return NULL; +} + bool DepsLog::Recompact(const string& path, string* err) { METRIC_RECORD(".ninja_deps recompact"); @@ -321,7 +331,7 @@ bool DepsLog::Recompact(const string& path, string* err) { // OpenForWrite() opens for append. Make sure it's not appending to a // left-over file from a previous recompaction attempt that crashed somehow. - unlink(temp_path.c_str()); + platformAwareUnlink(temp_path.c_str()); DepsLog new_log; if (!new_log.OpenForWrite(temp_path, err)) @@ -353,7 +363,7 @@ bool DepsLog::Recompact(const string& path, string* err) { deps_.swap(new_log.deps_); nodes_.swap(new_log.nodes_); - if (unlink(path.c_str()) < 0) { + if (platformAwareUnlink(path.c_str()) < 0) { *err = strerror(errno); return false; } @@ -366,7 +376,7 @@ bool DepsLog::Recompact(const string& path, string* err) { return true; } -bool DepsLog::IsDepsEntryLiveFor(Node* node) { +bool DepsLog::IsDepsEntryLiveFor(const Node* node) { // Skip entries that don't have in-edges or whose edges don't have a // "deps" attribute. They were in the deps log from previous builds, but // the the files they were for were removed from the build and their deps @@ -389,6 +399,7 @@ bool DepsLog::UpdateDeps(int out_id, Deps* deps) { bool DepsLog::RecordId(Node* node) { int path_size = node->path().size(); + assert(path_size > 0 && "Trying to record empty path Node!"); int padding = (4 - path_size % 4) % 4; // Pad path to 4 byte boundary. unsigned size = path_size + padding + 4; @@ -396,10 +407,13 @@ bool DepsLog::RecordId(Node* node) { errno = ERANGE; return false; } + + if (!OpenForWriteIfNeeded()) { + return false; + } if (fwrite(&size, 4, 1, file_) < 1) return false; if (fwrite(node->path().data(), path_size, 1, file_) < 1) { - assert(node->path().size() > 0); return false; } if (padding && fwrite("\0\0", padding, 1, file_) < 1) @@ -416,3 +430,37 @@ bool DepsLog::RecordId(Node* node) { return true; } + +bool DepsLog::OpenForWriteIfNeeded() { + if (file_path_.empty()) { + return true; + } + file_ = fopen(file_path_.c_str(), "ab"); + if (!file_) { + return false; + } + // Set the buffer size to this and flush the file buffer after every record + // to make sure records aren't written partially. + if (setvbuf(file_, NULL, _IOFBF, kMaxRecordSize + 1) != 0) { + return false; + } + SetCloseOnExec(fileno(file_)); + + // Opening a file in append mode doesn't set the file pointer to the file's + // end on Windows. Do that explicitly. + fseek(file_, 0, SEEK_END); + + if (ftell(file_) == 0) { + if (fwrite(kFileSignature, sizeof(kFileSignature) - 1, 1, file_) < 1) { + return false; + } + if (fwrite(&kCurrentVersion, 4, 1, file_) < 1) { + return false; + } + } + if (fflush(file_) != 0) { + return false; + } + file_path_.clear(); + return true; +} diff --git a/src/deps_log.h b/src/deps_log.h index 3812a28a80..cb82c25908 100644 --- a/src/deps_log.h +++ b/src/deps_log.h @@ -17,10 +17,10 @@ #include #include -using namespace std; #include +#include "load_status.h" #include "timestamp.h" struct Node; @@ -70,9 +70,10 @@ struct DepsLog { ~DepsLog(); // Writing (build-time) interface. - bool OpenForWrite(const string& path, string* err); - bool RecordDeps(Node* node, TimeStamp mtime, const vector& nodes); - bool RecordDeps(Node* node, TimeStamp mtime, int node_count, Node** nodes); + bool OpenForWrite(const std::string& path, std::string* err); + bool RecordDeps(Node* node, TimeStamp mtime, const std::vector& nodes); + bool RecordDeps(Node* node, TimeStamp mtime, int node_count, + Node* const* nodes); void Close(); // Reading (startup-time) interface. @@ -84,11 +85,12 @@ struct DepsLog { int node_count; Node** nodes; }; - bool Load(const string& path, State* state, string* err); + LoadStatus Load(const std::string& path, State* state, std::string* err); Deps* GetDeps(Node* node); + Node* GetFirstReverseDepsNode(Node* node); /// Rewrite the known log entries, throwing away old data. - bool Recompact(const string& path, string* err); + bool Recompact(const std::string& path, std::string* err); /// Returns if the deps entry for a node is still reachable from the manifest. /// @@ -96,11 +98,11 @@ struct DepsLog { /// past but are no longer part of the manifest. This function returns if /// this is the case for a given node. This function is slow, don't call /// it from code that runs on every build. - bool IsDepsEntryLiveFor(Node* node); + static bool IsDepsEntryLiveFor(const Node* node); /// Used for tests. - const vector& nodes() const { return nodes_; } - const vector& deps() const { return deps_; } + const std::vector& nodes() const { return nodes_; } + const std::vector& deps() const { return deps_; } private: // Updates the in-memory representation. Takes ownership of |deps|. @@ -109,13 +111,18 @@ struct DepsLog { // Write a node name record, assigning it an id. bool RecordId(Node* node); + /// Should be called before using file_. When false is returned, errno will + /// be set. + bool OpenForWriteIfNeeded(); + bool needs_recompaction_; FILE* file_; + std::string file_path_; /// Maps id -> Node. - vector nodes_; + std::vector nodes_; /// Maps id -> deps of that id. - vector deps_; + std::vector deps_; friend struct DepsLogTest; }; diff --git a/src/deps_log_test.cc b/src/deps_log_test.cc index 0cdeb45bed..61a9dcf029 100644 --- a/src/deps_log_test.cc +++ b/src/deps_log_test.cc @@ -19,10 +19,13 @@ #include #endif +#include "disk_interface.h" #include "graph.h" #include "util.h" #include "test.h" +using namespace std; + namespace { const char kTestFilename[] = "DepsLogTest-tempfile"; @@ -30,11 +33,9 @@ const char kTestFilename[] = "DepsLogTest-tempfile"; struct DepsLogTest : public testing::Test { virtual void SetUp() { // In case a crashing test left a stale file behind. - unlink(kTestFilename); - } - virtual void TearDown() { - unlink(kTestFilename); + platformAwareUnlink(kTestFilename); } + virtual void TearDown() { platformAwareUnlink(kTestFilename); } }; TEST_F(DepsLogTest, WriteRead) { @@ -136,9 +137,13 @@ TEST_F(DepsLogTest, DoubleEntry) { deps.push_back(state.GetNode("bar.h", 0)); log.RecordDeps(state.GetNode("out.o", 0), 1, deps); log.Close(); - +#ifdef __USE_LARGEFILE64 + struct stat64 st; + ASSERT_EQ(0, stat64(kTestFilename, &st)); +#else struct stat st; ASSERT_EQ(0, stat(kTestFilename, &st)); +#endif file_size = (int)st.st_size; ASSERT_GT(file_size, 0); } @@ -158,9 +163,13 @@ TEST_F(DepsLogTest, DoubleEntry) { deps.push_back(state.GetNode("bar.h", 0)); log.RecordDeps(state.GetNode("out.o", 0), 1, deps); log.Close(); - +#ifdef __USE_LARGEFILE64 + struct stat64 st; + ASSERT_EQ(0, stat64(kTestFilename, &st)); +#else struct stat st; ASSERT_EQ(0, stat(kTestFilename, &st)); +#endif int file_size_2 = (int)st.st_size; ASSERT_EQ(file_size, file_size_2); } @@ -196,9 +205,13 @@ TEST_F(DepsLogTest, Recompact) { log.RecordDeps(state.GetNode("other_out.o", 0), 1, deps); log.Close(); - +#ifdef __USE_LARGEFILE64 + struct stat64 st; + ASSERT_EQ(0, stat64(kTestFilename, &st)); +#else struct stat st; ASSERT_EQ(0, stat(kTestFilename, &st)); +#endif file_size = (int)st.st_size; ASSERT_GT(file_size, 0); } @@ -220,8 +233,13 @@ TEST_F(DepsLogTest, Recompact) { log.RecordDeps(state.GetNode("out.o", 0), 1, deps); log.Close(); +#ifdef __USE_LARGEFILE64 + struct stat64 st; + ASSERT_EQ(0, stat64(kTestFilename, &st)); +#else struct stat st; ASSERT_EQ(0, stat(kTestFilename, &st)); +#endif file_size_2 = (int)st.st_size; // The file should grow to record the new deps. ASSERT_GT(file_size_2, file_size); @@ -271,8 +289,13 @@ TEST_F(DepsLogTest, Recompact) { ASSERT_EQ(other_out, log.nodes()[other_out->id()]); // The file should have shrunk a bit for the smaller deps. +#ifdef __USE_LARGEFILE64 + struct stat64 st; + ASSERT_EQ(0, stat64(kTestFilename, &st)); +#else struct stat st; ASSERT_EQ(0, stat(kTestFilename, &st)); +#endif file_size_3 = (int)st.st_size; ASSERT_LT(file_size_3, file_size_2); } @@ -315,8 +338,13 @@ TEST_F(DepsLogTest, Recompact) { ASSERT_EQ(-1, state.LookupNode("baz.h")->id()); // The file should have shrunk more. +#ifdef __USE_LARGEFILE64 + struct stat64 st; + ASSERT_EQ(0, stat64(kTestFilename, &st)); +#else struct stat st; ASSERT_EQ(0, stat(kTestFilename, &st)); +#endif int file_size_4 = (int)st.st_size; ASSERT_LT(file_size_4, file_size_3); } @@ -372,8 +400,13 @@ TEST_F(DepsLogTest, Truncated) { } // Get the file size. +#ifdef __USE_LARGEFILE64 + struct stat64 st; + ASSERT_EQ(0, stat64(kTestFilename, &st)); +#else struct stat st; ASSERT_EQ(0, stat(kTestFilename, &st)); +#endif // Try reloading at truncated sizes. // Track how many nodes/deps were found; they should decrease with @@ -388,7 +421,7 @@ TEST_F(DepsLogTest, Truncated) { DepsLog log; EXPECT_TRUE(log.Load(kTestFilename, &state, &err)); if (!err.empty()) { - // At some point the log will be so short as to be unparseable. + // At some point the log will be so short as to be unparsable. break; } @@ -432,8 +465,13 @@ TEST_F(DepsLogTest, TruncatedRecovery) { // Shorten the file, corrupting the last record. { +#ifdef __USE_LARGEFILE64 + struct stat64 st; + ASSERT_EQ(0, stat64(kTestFilename, &st)); +#else struct stat st; ASSERT_EQ(0, stat(kTestFilename, &st)); +#endif string err; ASSERT_TRUE(Truncate(kTestFilename, st.st_size - 2, &err)); } @@ -476,4 +514,166 @@ TEST_F(DepsLogTest, TruncatedRecovery) { } } +TEST_F(DepsLogTest, ReverseDepsNodes) { + State state; + DepsLog log; + string err; + EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + + vector deps; + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar.h", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); + + deps.clear(); + deps.push_back(state.GetNode("foo.h", 0)); + deps.push_back(state.GetNode("bar2.h", 0)); + log.RecordDeps(state.GetNode("out2.o", 0), 2, deps); + + log.Close(); + + Node* rev_deps = log.GetFirstReverseDepsNode(state.GetNode("foo.h", 0)); + EXPECT_TRUE(rev_deps == state.GetNode("out.o", 0) || + rev_deps == state.GetNode("out2.o", 0)); + + rev_deps = log.GetFirstReverseDepsNode(state.GetNode("bar.h", 0)); + EXPECT_TRUE(rev_deps == state.GetNode("out.o", 0)); +} + +TEST_F(DepsLogTest, MalformedDepsLog) { + std::string err; + { + State state; + DepsLog log; + EXPECT_TRUE(log.OpenForWrite(kTestFilename, &err)); + ASSERT_EQ("", err); + + // First, create a valid log file. + std::vector deps; + deps.push_back(state.GetNode("foo.hh", 0)); + deps.push_back(state.GetNode("bar.hpp", 0)); + log.RecordDeps(state.GetNode("out.o", 0), 1, deps); + log.Close(); + } + + // Now read its value, validate it a little. + RealDiskInterface disk; + + std::string original_contents; + ASSERT_EQ(FileReader::Okay, disk.ReadFile(kTestFilename, + &original_contents, &err)); + + const size_t version_offset = 12; + ASSERT_EQ("# ninjadeps\n", original_contents.substr(0, version_offset)); + ASSERT_EQ('\x04', original_contents[version_offset + 0]); + ASSERT_EQ('\x00', original_contents[version_offset + 1]); + ASSERT_EQ('\x00', original_contents[version_offset + 2]); + ASSERT_EQ('\x00', original_contents[version_offset + 3]); + + // clang-format off + static const uint8_t kFirstRecord[] = { + // size field == 0x0000000c + 0x0c, 0x00, 0x00, 0x00, + // name field = 'out.o' + 3 bytes of padding. + 'o', 'u', 't', '.', 'o', 0x00, 0x00, 0x00, + // checksum = ~0 + 0xff, 0xff, 0xff, 0xff, + }; + // clang-format on + const size_t kFirstRecordLen = sizeof(kFirstRecord); + const size_t first_offset = version_offset + 4; + +#define COMPARE_RECORD(start_pos, reference, len) \ + ASSERT_EQ(original_contents.substr(start_pos, len), std::string(reinterpret_cast(reference), len)) + + COMPARE_RECORD(first_offset, kFirstRecord, kFirstRecordLen); + + const size_t second_offset = first_offset + kFirstRecordLen; + // clang-format off + static const uint8_t kSecondRecord[] = { + // size field == 0x0000000c + 0x0c, 0x00, 0x00, 0x00, + // name field = 'foo.hh' + 2 bytes of padding. + 'f', 'o', 'o', '.', 'h', 'h', 0x00, 0x00, + // checksum = ~1 + 0xfe, 0xff, 0xff, 0xff, + }; + // clang-format on + const size_t kSecondRecordLen = sizeof(kSecondRecord); + COMPARE_RECORD(second_offset, kSecondRecord, kSecondRecordLen); + + // Then start generating corrupted versions and trying to load them. + const char kBadLogFile[] = "DepsLogTest-corrupted.tempfile"; + + // Helper lambda to rewrite the bad log file with new content. + auto write_bad_log_file = + [&disk, &kBadLogFile](const std::string& bad_contents) -> bool { + (void)disk.RemoveFile(kBadLogFile); + return disk.WriteFile(kBadLogFile, bad_contents); + }; + + // First, corrupt the header. + std::string bad_contents = original_contents; + bad_contents[0] = '@'; + + ASSERT_TRUE(write_bad_log_file(bad_contents)) << strerror(errno); + { + State state; + DepsLog log; + err.clear(); + ASSERT_EQ(LOAD_SUCCESS, log.Load(kBadLogFile, &state, &err)); + ASSERT_EQ("bad deps log signature or version; starting over", err); + } + + // Second, truncate the version. + bad_contents = original_contents.substr(0, version_offset + 3); + ASSERT_TRUE(write_bad_log_file(bad_contents)) << strerror(errno); + { + State state; + DepsLog log; + err.clear(); + ASSERT_EQ(LOAD_SUCCESS, log.Load(kBadLogFile, &state, &err)); + ASSERT_EQ("bad deps log signature or version; starting over", err); + } + + // Truncate first record's |size| field. The loader should recover. + bad_contents = original_contents.substr(0, version_offset + 4 + 3); + ASSERT_TRUE(write_bad_log_file(bad_contents)) << strerror(errno); + { + State state; + DepsLog log; + err.clear(); + ASSERT_EQ(LOAD_SUCCESS, log.Load(kBadLogFile, &state, &err)); + ASSERT_EQ("", err); + } + + // Corrupt first record |size| value. + bad_contents = original_contents; + bad_contents[first_offset + 0] = '\x55'; + bad_contents[first_offset + 1] = '\xaa'; + bad_contents[first_offset + 2] = '\xff'; + bad_contents[first_offset + 3] = '\xff'; + ASSERT_TRUE(write_bad_log_file(bad_contents)) << strerror(errno); + { + State state; + DepsLog log; + err.clear(); + ASSERT_EQ(LOAD_SUCCESS, log.Load(kBadLogFile, &state, &err)); + ASSERT_EQ("premature end of file; recovering", err); + } + + // Make first record |size| less than 4. + bad_contents = original_contents; + bad_contents[first_offset] = '\x01'; + ASSERT_TRUE(write_bad_log_file(bad_contents)) << strerror(errno); + { + State state; + DepsLog log; + err.clear(); + ASSERT_EQ(LOAD_SUCCESS, log.Load(kBadLogFile, &state, &err)); + ASSERT_EQ("premature end of file; recovering", err); + } +} + } // anonymous namespace diff --git a/src/disk_interface.cc b/src/disk_interface.cc index d4c2fb0878..7f86e19a2f 100644 --- a/src/disk_interface.cc +++ b/src/disk_interface.cc @@ -23,14 +23,19 @@ #include #ifdef _WIN32 -#include -#include #include // _mkdir +#include + +#include +#else +#include #endif #include "metrics.h" #include "util.h" +using namespace std; + namespace { string DirName(const string& path) { @@ -106,7 +111,8 @@ bool StatAllFilesInDir(const string& dir, map* stamps, if (find_handle == INVALID_HANDLE_VALUE) { DWORD win_err = GetLastError(); - if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) + if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND || + win_err == ERROR_DIRECTORY) return true; *err = "FindFirstFileExA(" + dir + "): " + GetLastErrorString(); return false; @@ -152,13 +158,31 @@ bool DiskInterface::MakeDirs(const string& path) { } // RealDiskInterface ----------------------------------------------------------- +RealDiskInterface::RealDiskInterface() +#ifdef _WIN32 +: use_cache_(false), long_paths_enabled_(false) { + // Probe ntdll.dll for RtlAreLongPathsEnabled, and call it if it exists. + HINSTANCE ntdll_lib = ::GetModuleHandleW(L"ntdll"); + if (ntdll_lib) { + typedef BOOLEAN(WINAPI FunctionType)(); + auto* func_ptr = FunctionCast( + ::GetProcAddress(ntdll_lib, "RtlAreLongPathsEnabled")); + if (func_ptr) { + long_paths_enabled_ = (*func_ptr)(); + } + } +} +#else +{} +#endif TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { METRIC_RECORD("node stat"); #ifdef _WIN32 // MSDN: "Naming Files, Paths, and Namespaces" // http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx - if (!path.empty() && path[0] != '\\' && path.size() > MAX_PATH) { + if (!path.empty() && !AreLongPathsEnabled() && path[0] != '\\' && + path.size() > MAX_PATH) { ostringstream err_stream; err_stream << "Stat(" << path << "): Filename longer than " << MAX_PATH << " characters"; @@ -176,12 +200,13 @@ TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { dir = path; } - transform(dir.begin(), dir.end(), dir.begin(), ::tolower); + string dir_lowercase = dir; + transform(dir.begin(), dir.end(), dir_lowercase.begin(), ::tolower); transform(base.begin(), base.end(), base.begin(), ::tolower); - Cache::iterator ci = cache_.find(dir); + Cache::iterator ci = cache_.find(dir_lowercase); if (ci == cache_.end()) { - ci = cache_.insert(make_pair(dir, DirCache())).first; + ci = cache_.insert(make_pair(dir_lowercase, DirCache())).first; if (!StatAllFilesInDir(dir.empty() ? "." : dir, &ci->second, err)) { cache_.erase(ci); return -1; @@ -189,9 +214,14 @@ TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { } DirCache::iterator di = ci->second.find(base); return di != ci->second.end() ? di->second : 0; +#else +#ifdef __USE_LARGEFILE64 + struct stat64 st; + if (stat64(path.c_str(), &st) < 0) { #else struct stat st; if (stat(path.c_str(), &st) < 0) { +#endif if (errno == ENOENT || errno == ENOTDIR) return 0; *err = "stat(" + path + "): " + strerror(errno); @@ -202,19 +232,13 @@ TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { // that it doesn't exist. if (st.st_mtime == 0) return 1; -#if defined(__APPLE__) && !defined(_POSIX_C_SOURCE) +#if defined(_AIX) + return (int64_t)st.st_mtime * 1000000000LL + st.st_mtime_n; +#elif defined(__APPLE__) return ((int64_t)st.st_mtimespec.tv_sec * 1000000000LL + st.st_mtimespec.tv_nsec); -#elif (_POSIX_C_SOURCE >= 200809L || _XOPEN_SOURCE >= 700 || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || \ - defined(__BIONIC__) || (defined (__SVR4) && defined (__sun)) || defined(__FreeBSD__)) - // For glibc, see "Timestamp files" in the Notes of http://www.kernel.org/doc/man-pages/online/pages/man2/stat.2.html - // newlib, uClibc and musl follow the kernel (or Cygwin) headers and define the right macro values above. - // For bsd, see https://github.com/freebsd/freebsd/blob/master/sys/sys/stat.h and similar - // For bionic, C and POSIX API is always enabled. - // For solaris, see https://docs.oracle.com/cd/E88353_01/html/E37841/stat-2.html. +#elif defined(st_mtime) // A macro, so we're likely on modern POSIX. return (int64_t)st.st_mtim.tv_sec * 1000000000LL + st.st_mtim.tv_nsec; -#elif defined(_AIX) - return (int64_t)st.st_mtime * 1000000000LL + st.st_mtime_n; #else return (int64_t)st.st_mtime * 1000000000LL + st.st_mtimensec; #endif @@ -222,7 +246,7 @@ TimeStamp RealDiskInterface::Stat(const string& path, string* err) const { } bool RealDiskInterface::WriteFile(const string& path, const string& contents) { - FILE* fp = fopen(path.c_str(), "w"); + FILE* fp = fopen(path.c_str(), "wb"); if (fp == NULL) { Error("WriteFile(%s): Unable to create file. %s", path.c_str(), strerror(errno)); @@ -267,6 +291,47 @@ FileReader::Status RealDiskInterface::ReadFile(const string& path, } int RealDiskInterface::RemoveFile(const string& path) { +#ifdef _WIN32 + DWORD attributes = GetFileAttributesA(path.c_str()); + if (attributes == INVALID_FILE_ATTRIBUTES) { + DWORD win_err = GetLastError(); + if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) { + return 1; + } + } else if (attributes & FILE_ATTRIBUTE_READONLY) { + // On non-Windows systems, remove() will happily delete read-only files. + // On Windows Ninja should behave the same: + // https://github.com/ninja-build/ninja/issues/1886 + // Skip error checking. If this fails, accept whatever happens below. + SetFileAttributesA(path.c_str(), attributes & ~FILE_ATTRIBUTE_READONLY); + } + if (attributes & FILE_ATTRIBUTE_DIRECTORY) { + // remove() deletes both files and directories. On Windows we have to + // select the correct function (DeleteFile will yield Permission Denied when + // used on a directory) + // This fixes the behavior of ninja -t clean in some cases + // https://github.com/ninja-build/ninja/issues/828 + if (!RemoveDirectoryA(path.c_str())) { + DWORD win_err = GetLastError(); + if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) { + return 1; + } + // Report remove(), not RemoveDirectory(), for cross-platform consistency. + Error("remove(%s): %s", path.c_str(), GetLastErrorString().c_str()); + return -1; + } + } else { + if (!DeleteFileA(path.c_str())) { + DWORD win_err = GetLastError(); + if (win_err == ERROR_FILE_NOT_FOUND || win_err == ERROR_PATH_NOT_FOUND) { + return 1; + } + // Report as remove(), not DeleteFile(), for cross-platform consistency. + Error("remove(%s): %s", path.c_str(), GetLastErrorString().c_str()); + return -1; + } + } +#else if (remove(path.c_str()) < 0) { switch (errno) { case ENOENT: @@ -275,9 +340,9 @@ int RealDiskInterface::RemoveFile(const string& path) { Error("remove(%s): %s", path.c_str(), strerror(errno)); return -1; } - } else { - return 0; } +#endif + return 0; } void RealDiskInterface::AllowStatCache(bool allow) { @@ -287,3 +352,9 @@ void RealDiskInterface::AllowStatCache(bool allow) { cache_.clear(); #endif } + +#ifdef _WIN32 +bool RealDiskInterface::AreLongPathsEnabled(void) const { + return long_paths_enabled_; +} +#endif diff --git a/src/disk_interface.h b/src/disk_interface.h index 145e0892f4..74200b8f5b 100644 --- a/src/disk_interface.h +++ b/src/disk_interface.h @@ -17,7 +17,6 @@ #include #include -using namespace std; #include "timestamp.h" @@ -35,8 +34,8 @@ struct FileReader { /// Read and store in given string. On success, return Okay. /// On error, return another Status and fill |err|. - virtual Status ReadFile(const string& path, string* contents, - string* err) = 0; + virtual Status ReadFile(const std::string& path, std::string* contents, + std::string* err) = 0; }; /// Interface for accessing the disk. @@ -46,53 +45,59 @@ struct FileReader { struct DiskInterface: public FileReader { /// stat() a file, returning the mtime, or 0 if missing and -1 on /// other errors. - virtual TimeStamp Stat(const string& path, string* err) const = 0; + virtual TimeStamp Stat(const std::string& path, std::string* err) const = 0; /// Create a directory, returning false on failure. - virtual bool MakeDir(const string& path) = 0; + virtual bool MakeDir(const std::string& path) = 0; /// Create a file, with the specified name and contents /// Returns true on success, false on failure - virtual bool WriteFile(const string& path, const string& contents) = 0; + virtual bool WriteFile(const std::string& path, + const std::string& contents) = 0; /// Remove the file named @a path. It behaves like 'rm -f path' so no errors /// are reported if it does not exists. /// @returns 0 if the file has been removed, /// 1 if the file does not exist, and /// -1 if an error occurs. - virtual int RemoveFile(const string& path) = 0; + virtual int RemoveFile(const std::string& path) = 0; /// Create all the parent directories for path; like mkdir -p /// `basename path`. - bool MakeDirs(const string& path); + bool MakeDirs(const std::string& path); }; /// Implementation of DiskInterface that actually hits the disk. struct RealDiskInterface : public DiskInterface { - RealDiskInterface() -#ifdef _WIN32 - : use_cache_(false) -#endif - {} + RealDiskInterface(); virtual ~RealDiskInterface() {} - virtual TimeStamp Stat(const string& path, string* err) const; - virtual bool MakeDir(const string& path); - virtual bool WriteFile(const string& path, const string& contents); - virtual Status ReadFile(const string& path, string* contents, string* err); - virtual int RemoveFile(const string& path); + virtual TimeStamp Stat(const std::string& path, std::string* err) const; + virtual bool MakeDir(const std::string& path); + virtual bool WriteFile(const std::string& path, const std::string& contents); + virtual Status ReadFile(const std::string& path, std::string* contents, + std::string* err); + virtual int RemoveFile(const std::string& path); /// Whether stat information can be cached. Only has an effect on Windows. void AllowStatCache(bool allow); +#ifdef _WIN32 + /// Whether long paths are enabled. Only has an effect on Windows. + bool AreLongPathsEnabled() const; +#endif + private: #ifdef _WIN32 /// Whether stat information can be cached. bool use_cache_; - typedef map DirCache; + /// Whether long paths are enabled. + bool long_paths_enabled_; + + typedef std::map DirCache; // TODO: Neither a map nor a hashmap seems ideal here. If the statcache // works out, come up with a better data structure. - typedef map Cache; + typedef std::map Cache; mutable Cache cache_; #endif }; diff --git a/src/disk_interface_test.cc b/src/disk_interface_test.cc index bac515d235..b09ed046e8 100644 --- a/src/disk_interface_test.cc +++ b/src/disk_interface_test.cc @@ -17,12 +17,15 @@ #ifdef _WIN32 #include #include +#include #endif #include "disk_interface.h" #include "graph.h" #include "test.h" +using namespace std; + namespace { struct DiskInterfaceTest : public testing::Test { @@ -63,6 +66,17 @@ TEST_F(DiskInterfaceTest, StatMissingFile) { EXPECT_EQ("", err); } +TEST_F(DiskInterfaceTest, StatMissingFileWithCache) { + disk_.AllowStatCache(true); + string err; + + // On Windows, the errno for FindFirstFileExA, which is used when the stat + // cache is enabled, is different when the directory name is not a directory. + ASSERT_TRUE(Touch("notadir")); + EXPECT_EQ(0, disk_.Stat("notadir/nosuchfile", &err)); + EXPECT_EQ("", err); +} + TEST_F(DiskInterfaceTest, StatBadPath) { string err; #ifdef _WIN32 @@ -83,6 +97,24 @@ TEST_F(DiskInterfaceTest, StatExistingFile) { EXPECT_EQ("", err); } +#ifdef _WIN32 +TEST_F(DiskInterfaceTest, StatExistingFileWithLongPath) { + string err; + char currentdir[32767]; + _getcwd(currentdir, sizeof(currentdir)); + const string filename = string(currentdir) + +"\\filename_with_256_characters_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\ +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\ +xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx\ +xxxxxxxxxxxxxxxxxxxxx"; + const string prefixed = "\\\\?\\" + filename; + ASSERT_TRUE(Touch(prefixed.c_str())); + EXPECT_GT(disk_.Stat(disk_.AreLongPathsEnabled() ? + filename : prefixed, &err), 1); + EXPECT_EQ("", err); +} +#endif + TEST_F(DiskInterfaceTest, StatExistingDir) { string err; ASSERT_TRUE(disk_.MakeDir("subdir")); @@ -190,13 +222,13 @@ TEST_F(DiskInterfaceTest, ReadFile) { TEST_F(DiskInterfaceTest, MakeDirs) { string path = "path/with/double//slash/"; - EXPECT_TRUE(disk_.MakeDirs(path.c_str())); + EXPECT_TRUE(disk_.MakeDirs(path)); FILE* f = fopen((path + "a_file").c_str(), "w"); EXPECT_TRUE(f); EXPECT_EQ(0, fclose(f)); #ifdef _WIN32 string path2 = "another\\with\\back\\\\slashes\\"; - EXPECT_TRUE(disk_.MakeDirs(path2.c_str())); + EXPECT_TRUE(disk_.MakeDirs(path2)); FILE* f2 = fopen((path2 + "a_file").c_str(), "w"); EXPECT_TRUE(f2); EXPECT_EQ(0, fclose(f2)); @@ -209,11 +241,25 @@ TEST_F(DiskInterfaceTest, RemoveFile) { EXPECT_EQ(0, disk_.RemoveFile(kFileName)); EXPECT_EQ(1, disk_.RemoveFile(kFileName)); EXPECT_EQ(1, disk_.RemoveFile("does not exist")); +#ifdef _WIN32 + ASSERT_TRUE(Touch(kFileName)); + EXPECT_EQ(0, system((std::string("attrib +R ") + kFileName).c_str())); + EXPECT_EQ(0, disk_.RemoveFile(kFileName)); + EXPECT_EQ(1, disk_.RemoveFile(kFileName)); +#endif +} + +TEST_F(DiskInterfaceTest, RemoveDirectory) { + const char* kDirectoryName = "directory-to-remove"; + EXPECT_TRUE(disk_.MakeDir(kDirectoryName)); + EXPECT_EQ(0, disk_.RemoveFile(kDirectoryName)); + EXPECT_EQ(1, disk_.RemoveFile(kDirectoryName)); + EXPECT_EQ(1, disk_.RemoveFile("does not exist")); } struct StatTest : public StateTestWithBuiltinRules, public DiskInterface { - StatTest() : scan_(&state_, NULL, NULL, this, NULL) {} + StatTest() : scan_(&state_, NULL, NULL, this, NULL, NULL) {} // DiskInterface implementation. virtual TimeStamp Stat(const string& path, string* err) const; @@ -256,7 +302,7 @@ TEST_F(StatTest, Simple) { EXPECT_TRUE(out->Stat(this, &err)); EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); - scan_.RecomputeDirty(out, NULL); + scan_.RecomputeDirty(out, NULL, NULL); ASSERT_EQ(2u, stats_.size()); ASSERT_EQ("out", stats_[0]); ASSERT_EQ("in", stats_[1]); @@ -272,7 +318,7 @@ TEST_F(StatTest, TwoStep) { EXPECT_TRUE(out->Stat(this, &err)); EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); - scan_.RecomputeDirty(out, NULL); + scan_.RecomputeDirty(out, NULL, NULL); ASSERT_EQ(3u, stats_.size()); ASSERT_EQ("out", stats_[0]); ASSERT_TRUE(GetNode("out")->dirty()); @@ -292,7 +338,7 @@ TEST_F(StatTest, Tree) { EXPECT_TRUE(out->Stat(this, &err)); EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); - scan_.RecomputeDirty(out, NULL); + scan_.RecomputeDirty(out, NULL, NULL); ASSERT_EQ(1u + 6u, stats_.size()); ASSERT_EQ("mid1", stats_[1]); ASSERT_TRUE(GetNode("mid1")->dirty()); @@ -313,7 +359,7 @@ TEST_F(StatTest, Middle) { EXPECT_TRUE(out->Stat(this, &err)); EXPECT_EQ("", err); ASSERT_EQ(1u, stats_.size()); - scan_.RecomputeDirty(out, NULL); + scan_.RecomputeDirty(out, NULL, NULL); ASSERT_FALSE(GetNode("in")->dirty()); ASSERT_TRUE(GetNode("mid")->dirty()); ASSERT_TRUE(GetNode("out")->dirty()); diff --git a/src/dyndep.cc b/src/dyndep.cc index 2aee601b59..15e69841b6 100644 --- a/src/dyndep.cc +++ b/src/dyndep.cc @@ -20,10 +20,13 @@ #include "debug_flags.h" #include "disk_interface.h" #include "dyndep_parser.h" +#include "explanations.h" #include "graph.h" #include "state.h" #include "util.h" +using namespace std; + bool DyndepLoader::LoadDyndeps(Node* node, std::string* err) const { DyndepFile ddf; return LoadDyndeps(node, &ddf, err); @@ -35,15 +38,14 @@ bool DyndepLoader::LoadDyndeps(Node* node, DyndepFile* ddf, node->set_dyndep_pending(false); // Load the dyndep information from the file. - EXPLAIN("loading dyndep file '%s'", node->path().c_str()); + explanations_.Record(node, "loading dyndep file '%s'", node->path().c_str()); + if (!LoadDyndepFile(node, ddf, err)) return false; // Update each edge that specified this node as its dyndep binding. std::vector const& out_edges = node->out_edges(); - for (std::vector::const_iterator oe = out_edges.begin(); - oe != out_edges.end(); ++oe) { - Edge* const edge = *oe; + for (Edge* edge : out_edges) { if (edge->dyndep_ != node) continue; @@ -63,10 +65,9 @@ bool DyndepLoader::LoadDyndeps(Node* node, DyndepFile* ddf, } // Reject extra outputs in dyndep file. - for (DyndepFile::const_iterator oe = ddf->begin(); oe != ddf->end(); - ++oe) { - if (!oe->second.used_) { - Edge* const edge = oe->first; + for (const auto& dyndep_output : *ddf) { + if (!dyndep_output.second.used_) { + Edge* const edge = dyndep_output.first; *err = ("dyndep file '" + node->path() + "' mentions output " "'" + edge->outputs_[0]->path() + "' whose build statement " "does not have a dyndep binding for the file"); @@ -92,14 +93,13 @@ bool DyndepLoader::UpdateEdge(Edge* edge, Dyndeps const* dyndeps, edge->implicit_outs_ += dyndeps->implicit_outputs_.size(); // Add this edge as incoming to each new output. - for (std::vector::const_iterator i = - dyndeps->implicit_outputs_.begin(); - i != dyndeps->implicit_outputs_.end(); ++i) { - if ((*i)->in_edge() != NULL) { - *err = "multiple rules generate " + (*i)->path(); + for (Node* node : dyndeps->implicit_outputs_) { + if (node->in_edge()) { + // This node already has an edge producing it. + *err = "multiple rules generate " + node->path(); return false; } - (*i)->set_in_edge(edge); + node->set_in_edge(edge); } // Add the dyndep-discovered inputs to the edge. @@ -109,10 +109,8 @@ bool DyndepLoader::UpdateEdge(Edge* edge, Dyndeps const* dyndeps, edge->implicit_deps_ += dyndeps->implicit_inputs_.size(); // Add this edge as outgoing from each new input. - for (std::vector::const_iterator i = - dyndeps->implicit_inputs_.begin(); - i != dyndeps->implicit_inputs_.end(); ++i) - (*i)->AddOutEdge(edge); + for (Node* node : dyndeps->implicit_inputs_) + node->AddOutEdge(edge); return true; } diff --git a/src/dyndep.h b/src/dyndep.h index 907f921d17..5999800ef3 100644 --- a/src/dyndep.h +++ b/src/dyndep.h @@ -19,6 +19,8 @@ #include #include +#include "explanations.h" + struct DiskInterface; struct Edge; struct Node; @@ -42,8 +44,10 @@ struct DyndepFile: public std::map {}; /// DyndepLoader loads dynamically discovered dependencies, as /// referenced via the "dyndep" attribute in build files. struct DyndepLoader { - DyndepLoader(State* state, DiskInterface* disk_interface) - : state_(state), disk_interface_(disk_interface) {} + DyndepLoader(State* state, DiskInterface* disk_interface, + Explanations* explanations = nullptr) + : state_(state), disk_interface_(disk_interface), + explanations_(explanations) {} /// Load a dyndep file from the given node's path and update the /// build graph with the new information. One overload accepts @@ -59,6 +63,7 @@ struct DyndepLoader { State* state_; DiskInterface* disk_interface_; + mutable OptionalExplanations explanations_; }; #endif // NINJA_DYNDEP_LOADER_H_ diff --git a/src/dyndep_parser.cc b/src/dyndep_parser.cc index baebbaceee..f35f48d92d 100644 --- a/src/dyndep_parser.cc +++ b/src/dyndep_parser.cc @@ -22,6 +22,8 @@ #include "util.h" #include "version.h" +using namespace std; + DyndepParser::DyndepParser(State* state, FileReader* file_reader, DyndepFile* dyndep_file) : Parser(state, file_reader) @@ -86,7 +88,6 @@ bool DyndepParser::ParseDyndepVersion(string* err) { if (major != 1 || minor != 0) { return lexer_.Error( string("unsupported 'ninja_dyndep_version = ") + version + "'", err); - return false; } return true; } @@ -94,11 +95,7 @@ bool DyndepParser::ParseDyndepVersion(string* err) { bool DyndepParser::ParseLet(string* key, EvalString* value, string* err) { if (!lexer_.ReadIdent(key)) return lexer_.Error("expected variable name", err); - if (!ExpectToken(Lexer::EQUALS, err)) - return false; - if (!lexer_.ReadVarValue(value, err)) - return false; - return true; + return (ExpectToken(Lexer::EQUALS, err) && lexer_.ReadVarValue(value, err)); } bool DyndepParser::ParseEdge(string* err) { @@ -113,10 +110,10 @@ bool DyndepParser::ParseEdge(string* err) { return lexer_.Error("expected path", err); string path = out0.Evaluate(&env_); - string path_err; + if (path.empty()) + return lexer_.Error("empty path", err); uint64_t slash_bits; - if (!CanonicalizePath(&path, &slash_bits, &path_err)) - return lexer_.Error(path_err, err); + CanonicalizePath(&path, &slash_bits); Node* node = state_->LookupNode(path); if (!node || !node->in_edge()) return lexer_.Error("no build statement exists for '" + path + "'", err); @@ -198,23 +195,23 @@ bool DyndepParser::ParseEdge(string* err) { } dyndeps->implicit_inputs_.reserve(ins.size()); - for (vector::iterator i = ins.begin(); i != ins.end(); ++i) { - string path = i->Evaluate(&env_); - string path_err; + for (const EvalString& in : ins) { + string path = in.Evaluate(&env_); + if (path.empty()) + return lexer_.Error("empty path", err); uint64_t slash_bits; - if (!CanonicalizePath(&path, &slash_bits, &path_err)) - return lexer_.Error(path_err, err); + CanonicalizePath(&path, &slash_bits); Node* n = state_->GetNode(path, slash_bits); dyndeps->implicit_inputs_.push_back(n); } dyndeps->implicit_outputs_.reserve(outs.size()); - for (vector::iterator i = outs.begin(); i != outs.end(); ++i) { - string path = i->Evaluate(&env_); - string path_err; + for (const EvalString& out : outs) { + string path = out.Evaluate(&env_); + if (path.empty()) + return lexer_.Error("empty path", err); uint64_t slash_bits; - if (!CanonicalizePath(&path, &slash_bits, &path_err)) - return lexer_.Error(path_err, err); + CanonicalizePath(&path, &slash_bits); Node* n = state_->GetNode(path, slash_bits); dyndeps->implicit_outputs_.push_back(n); } diff --git a/src/dyndep_parser.h b/src/dyndep_parser.h index 09a3722ede..8f4c28de54 100644 --- a/src/dyndep_parser.h +++ b/src/dyndep_parser.h @@ -27,17 +27,18 @@ struct DyndepParser: public Parser { DyndepFile* dyndep_file); /// Parse a text string of input. Used by tests. - bool ParseTest(const string& input, string* err) { + bool ParseTest(const std::string& input, std::string* err) { return Parse("input", input, err); } private: /// Parse a file, given its contents as a string. - bool Parse(const string& filename, const string& input, string* err); + bool Parse(const std::string& filename, const std::string& input, + std:: string* err); - bool ParseDyndepVersion(string* err); - bool ParseLet(string* key, EvalString* val, string* err); - bool ParseEdge(string* err); + bool ParseDyndepVersion(std::string* err); + bool ParseLet(std::string* key, EvalString* val, std::string* err); + bool ParseEdge(std::string* err); DyndepFile* dyndep_file_; BindingEnv env_; diff --git a/src/dyndep_parser_test.cc b/src/dyndep_parser_test.cc index 39ec6574f2..1bba7ba840 100644 --- a/src/dyndep_parser_test.cc +++ b/src/dyndep_parser_test.cc @@ -22,6 +22,8 @@ #include "state.h" #include "test.h" +using namespace std; + struct DyndepParserTest : public testing::Test { void AssertParse(const char* input) { DyndepParser parser(&state_, &fs_, &dyndep_file_); diff --git a/src/edit_distance.cc b/src/edit_distance.cc index 3bb62b8589..34bf0e51ff 100644 --- a/src/edit_distance.cc +++ b/src/edit_distance.cc @@ -17,6 +17,8 @@ #include #include +using namespace std; + int EditDistance(const StringPiece& s1, const StringPiece& s2, bool allow_replacements, diff --git a/src/elide_middle.cc b/src/elide_middle.cc new file mode 100644 index 0000000000..cf17da5ba4 --- /dev/null +++ b/src/elide_middle.cc @@ -0,0 +1,276 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "elide_middle.h" + +#include +#include + +// Convenience class used to iterate over the ANSI color sequences +// of an input string. Note that this ignores non-color related +// ANSI sequences. Usage is: +// +// - Create instance, passing the input string to the constructor. +// - Loop over each sequence with: +// +// AnsiColorSequenceIterator iter; +// while (iter.HasSequence()) { +// .. use iter.SequenceStart() and iter.SequenceEnd() +// iter.NextSequence(); +// } +// +struct AnsiColorSequenceIterator { + // Constructor takes input string . + AnsiColorSequenceIterator(const std::string& input) + : input_(input.data()), input_end_(input_ + input.size()) { + FindNextSequenceFrom(input_); + } + + // Return true if an ANSI sequence was found. + bool HasSequence() const { return cur_end_ != 0; } + + // Start of the current sequence. + size_t SequenceStart() const { return cur_start_; } + + // End of the current sequence (index of the first character + // following the sequence). + size_t SequenceEnd() const { return cur_end_; } + + // Size of the current sequence in characters. + size_t SequenceSize() const { return cur_end_ - cur_start_; } + + // Returns true if |input_index| belongs to the current sequence. + bool SequenceContains(size_t input_index) const { + return (input_index >= cur_start_ && input_index < cur_end_); + } + + // Find the next sequence, if any, from the input. + // Returns false is there is no more sequence. + bool NextSequence() { + if (FindNextSequenceFrom(input_ + cur_end_)) + return true; + + cur_start_ = 0; + cur_end_ = 0; + return false; + } + + // Reset iterator to start of input. + void Reset() { + cur_start_ = cur_end_ = 0; + FindNextSequenceFrom(input_); + } + + private: + // Find the next sequence from the input, |from| being the starting position + // for the search, and must be in the [input_, input_end_] interval. On + // success, returns true after setting cur_start_ and cur_end_, on failure, + // return false. + bool FindNextSequenceFrom(const char* from) { + assert(from >= input_ && from <= input_end_); + auto* seq = + static_cast(::memchr(from, '\x1b', input_end_ - from)); + if (!seq) + return false; + + // The smallest possible color sequence if '\x1c[0m` and has four + // characters. + if (seq + 4 > input_end_) + return false; + + if (seq[1] != '[') + return FindNextSequenceFrom(seq + 1); + + // Skip parameters (digits + ; separator) + auto is_parameter_char = [](char ch) -> bool { + return (ch >= '0' && ch <= '9') || ch == ';'; + }; + + const char* end = seq + 2; + while (is_parameter_char(end[0])) { + if (++end == input_end_) + return false; // Incomplete sequence (no command). + } + + if (*end++ != 'm') { + // Not a color sequence. Restart the search after the first + // character following the [, in case this was a 3-char ANSI + // sequence (which is ignored here). + return FindNextSequenceFrom(seq + 3); + } + + // Found it! + cur_start_ = seq - input_; + cur_end_ = end - input_; + return true; + } + + size_t cur_start_ = 0; + size_t cur_end_ = 0; + const char* input_; + const char* input_end_; +}; + +// A class used to iterate over all characters of an input string, +// and return its visible position in the terminal, and whether that +// specific character is visible (or otherwise part of an ANSI color sequence). +// +// Example sequence and iterations, where 'ANSI' represents an ANSI Color +// sequence, and | is used to express concatenation +// +// |abcd|ANSI|efgh|ANSI|ijk| input string +// +// 11 1111 111 +// 0123 4567 8901 2345 678 input indices +// +// 1 +// 0123 4444 4567 8888 890 visible positions +// +// TTTT FFFF TTTT FFFF TTT is_visible +// +// Usage is: +// +// VisibleInputCharsIterator iter(input); +// while (iter.HasChar()) { +// ... use iter.InputIndex() to get input index of current char. +// ... use iter.VisiblePosition() to get its visible position. +// ... use iter.IsVisible() to check whether the current char is visible. +// +// NextChar(); +// } +// +struct VisibleInputCharsIterator { + VisibleInputCharsIterator(const std::string& input) + : input_size_(input.size()), ansi_iter_(input) {} + + // Return true if there is a character in the sequence. + bool HasChar() const { return input_index_ < input_size_; } + + // Return current input index. + size_t InputIndex() const { return input_index_; } + + // Return current visible position. + size_t VisiblePosition() const { return visible_pos_; } + + // Return true if the current input character is visible + // (i.e. not part of an ANSI color sequence). + bool IsVisible() const { return !ansi_iter_.SequenceContains(input_index_); } + + // Find next character from the input. + void NextChar() { + visible_pos_ += IsVisible(); + if (++input_index_ == ansi_iter_.SequenceEnd()) { + ansi_iter_.NextSequence(); + } + } + + private: + size_t input_size_; + size_t input_index_ = 0; + size_t visible_pos_ = 0; + AnsiColorSequenceIterator ansi_iter_; +}; + +void ElideMiddleInPlace(std::string& str, size_t max_width) { + if (str.size() <= max_width) { + return; + } + // Look for an ESC character. If there is none, use a fast path + // that avoids any intermediate allocations. + if (str.find('\x1b') == std::string::npos) { + const int ellipsis_width = 3; // Space for "...". + + // If max width is too small, do not keep anything from the input. + if (max_width <= ellipsis_width) { + str.assign("...", max_width); + return; + } + + // Keep only |max_width - ellipsis_size| visible characters from the input + // which will be split into two spans separated by "...". + const size_t remaining_size = max_width - ellipsis_width; + const size_t left_span_size = remaining_size / 2; + const size_t right_span_size = remaining_size - left_span_size; + + // Replace the gap in the input between the spans with "..." + const size_t gap_start = left_span_size; + const size_t gap_end = str.size() - right_span_size; + str.replace(gap_start, gap_end - gap_start, "..."); + return; + } + + // Compute visible width. + size_t visible_width = str.size(); + for (AnsiColorSequenceIterator ansi(str); ansi.HasSequence(); + ansi.NextSequence()) { + visible_width -= ansi.SequenceSize(); + } + + if (visible_width <= max_width) + return; + + // Compute the widths of the ellipsis, left span and right span + // visible space. + const size_t ellipsis_width = max_width < 3 ? max_width : 3; + const size_t visible_left_span_size = (max_width - ellipsis_width) / 2; + const size_t visible_right_span_size = + (max_width - ellipsis_width) - visible_left_span_size; + + // Compute the gap of visible characters that will be replaced by + // the ellipsis in visible space. + const size_t visible_gap_start = visible_left_span_size; + const size_t visible_gap_end = visible_width - visible_right_span_size; + + std::string result; + result.reserve(str.size()); + + // Parse the input chars info to: + // + // 1) Append any characters belonging to the left span (visible or not). + // + // 2) Add the ellipsis ("..." truncated to ellipsis_width). + // Note that its color is inherited from the left span chars + // which will never end with an ANSI sequence. + // + // 3) Append any ANSI sequence that appears inside the gap. This + // ensures the characters after the ellipsis appear with + // the right color, + // + // 4) Append any remaining characters (visible or not) to the result. + // + VisibleInputCharsIterator iter(str); + + // Step 1 - determine left span length in input chars. + for (; iter.HasChar(); iter.NextChar()) { + if (iter.VisiblePosition() == visible_gap_start) + break; + } + result.append(str.begin(), str.begin() + iter.InputIndex()); + + // Step 2 - Append the possibly-truncated ellipsis. + result.append("...", ellipsis_width); + + // Step 3 - Append elided ANSI sequences to the result. + for (; iter.HasChar(); iter.NextChar()) { + if (iter.VisiblePosition() == visible_gap_end) + break; + if (!iter.IsVisible()) + result.push_back(str[iter.InputIndex()]); + } + + // Step 4 - Append anything else. + result.append(str.begin() + iter.InputIndex(), str.end()); + + str = std::move(result); +} diff --git a/src/elide_middle.h b/src/elide_middle.h new file mode 100644 index 0000000000..128a9974e4 --- /dev/null +++ b/src/elide_middle.h @@ -0,0 +1,27 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_ELIDE_MIDDLE_H_ +#define NINJA_ELIDE_MIDDLE_H_ + +#include +#include + +/// Elide the given string @a str with '...' in the middle if the length +/// exceeds @a max_width. Note that this handles ANSI color sequences +/// properly (non-color related sequences are ignored, but using them +/// would wreak the cursor position or terminal state anyway). +void ElideMiddleInPlace(std::string& str, size_t max_width); + +#endif // NINJA_ELIDE_MIDDLE_H_ diff --git a/src/elide_middle_perftest.cc b/src/elide_middle_perftest.cc new file mode 100644 index 0000000000..94a8ccb944 --- /dev/null +++ b/src/elide_middle_perftest.cc @@ -0,0 +1,72 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include + +#include + +#include "elide_middle.h" +#include "metrics.h" + +static const char* kTestInputs[] = { + "01234567890123456789", + "012345\x1B[0;35m67890123456789", + "abcd\x1b[1;31mefg\x1b[0mhlkmnopqrstuvwxyz", +}; + +int main() { + std::vector times; + + int64_t kMaxTimeMillis = 5 * 1000; + int64_t base_time = GetTimeMillis(); + + const int kRuns = 100; + for (int j = 0; j < kRuns; ++j) { + int64_t start = GetTimeMillis(); + if (start >= base_time + kMaxTimeMillis) + break; + + const int kNumRepetitions = 2000; + for (int count = kNumRepetitions; count > 0; --count) { + for (const char* input : kTestInputs) { + size_t input_len = ::strlen(input); + for (size_t max_width = input_len; max_width > 0; --max_width) { + std::string str(input, input_len); + ElideMiddleInPlace(str, max_width); + } + } + } + + int delta = (int)(GetTimeMillis() - start); + times.push_back(delta); + } + + int min = times[0]; + int max = times[0]; + float total = 0; + for (size_t i = 0; i < times.size(); ++i) { + total += times[i]; + if (times[i] < min) + min = times[i]; + else if (times[i] > max) + max = times[i]; + } + + printf("min %dms max %dms avg %.1fms\n", min, max, total / times.size()); + + return 0; +} diff --git a/src/elide_middle_test.cc b/src/elide_middle_test.cc new file mode 100644 index 0000000000..ed80e4eb73 --- /dev/null +++ b/src/elide_middle_test.cc @@ -0,0 +1,101 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "elide_middle.h" + +#include "test.h" + +namespace { + +std::string ElideMiddle(const std::string& str, size_t width) { + std::string result = str; + ElideMiddleInPlace(result, width); + return result; +} + +} // namespace + + +TEST(ElideMiddle, NothingToElide) { + std::string input = "Nothing to elide in this short string."; + EXPECT_EQ(input, ElideMiddle(input, 80)); + EXPECT_EQ(input, ElideMiddle(input, 38)); + EXPECT_EQ("", ElideMiddle(input, 0)); + EXPECT_EQ(".", ElideMiddle(input, 1)); + EXPECT_EQ("..", ElideMiddle(input, 2)); + EXPECT_EQ("...", ElideMiddle(input, 3)); +} + +TEST(ElideMiddle, ElideInTheMiddle) { + std::string input = "01234567890123456789"; + EXPECT_EQ("...9", ElideMiddle(input, 4)); + EXPECT_EQ("0...9", ElideMiddle(input, 5)); + EXPECT_EQ("012...789", ElideMiddle(input, 9)); + EXPECT_EQ("012...6789", ElideMiddle(input, 10)); + EXPECT_EQ("0123...6789", ElideMiddle(input, 11)); + EXPECT_EQ("01234567...23456789", ElideMiddle(input, 19)); + EXPECT_EQ("01234567890123456789", ElideMiddle(input, 20)); +} + +// A few ANSI escape sequences. These macros make the following +// test easier to read and understand. +#define MAGENTA "\x1B[0;35m" +#define NOTHING "\33[m" +#define RED "\x1b[1;31m" +#define RESET "\x1b[0m" + +TEST(ElideMiddle, ElideAnsiEscapeCodes) { + std::string input = "012345" MAGENTA "67890123456789"; + EXPECT_EQ("012..." MAGENTA "6789", ElideMiddle(input, 10)); + EXPECT_EQ("012345" MAGENTA "67...23456789", ElideMiddle(input, 19)); + + EXPECT_EQ("Nothing " NOTHING " string.", + ElideMiddle("Nothing " NOTHING " string.", 18)); + EXPECT_EQ("0" NOTHING "12...6789", + ElideMiddle("0" NOTHING "1234567890123456789", 10)); + + input = "abcd" RED "efg" RESET "hlkmnopqrstuvwxyz"; + EXPECT_EQ("" RED RESET, ElideMiddle(input, 0)); + EXPECT_EQ("." RED RESET, ElideMiddle(input, 1)); + EXPECT_EQ(".." RED RESET, ElideMiddle(input, 2)); + EXPECT_EQ("..." RED RESET, ElideMiddle(input, 3)); + EXPECT_EQ("..." RED RESET "z", ElideMiddle(input, 4)); + EXPECT_EQ("a..." RED RESET "z", ElideMiddle(input, 5)); + EXPECT_EQ("a..." RED RESET "yz", ElideMiddle(input, 6)); + EXPECT_EQ("ab..." RED RESET "yz", ElideMiddle(input, 7)); + EXPECT_EQ("ab..." RED RESET "xyz", ElideMiddle(input, 8)); + EXPECT_EQ("abc..." RED RESET "xyz", ElideMiddle(input, 9)); + EXPECT_EQ("abc..." RED RESET "wxyz", ElideMiddle(input, 10)); + EXPECT_EQ("abcd..." RED RESET "wxyz", ElideMiddle(input, 11)); + EXPECT_EQ("abcd..." RED RESET "vwxyz", ElideMiddle(input, 12)); + + EXPECT_EQ("abcd" RED "ef..." RESET "uvwxyz", ElideMiddle(input, 15)); + EXPECT_EQ("abcd" RED "ef..." RESET "tuvwxyz", ElideMiddle(input, 16)); + EXPECT_EQ("abcd" RED "efg..." RESET "tuvwxyz", ElideMiddle(input, 17)); + EXPECT_EQ("abcd" RED "efg..." RESET "stuvwxyz", ElideMiddle(input, 18)); + EXPECT_EQ("abcd" RED "efg" RESET "h...stuvwxyz", ElideMiddle(input, 19)); + + input = "abcdef" RED "A" RESET "BC"; + EXPECT_EQ("..." RED RESET "C", ElideMiddle(input, 4)); + EXPECT_EQ("a..." RED RESET "C", ElideMiddle(input, 5)); + EXPECT_EQ("a..." RED RESET "BC", ElideMiddle(input, 6)); + EXPECT_EQ("ab..." RED RESET "BC", ElideMiddle(input, 7)); + EXPECT_EQ("ab..." RED "A" RESET "BC", ElideMiddle(input, 8)); + EXPECT_EQ("abcdef" RED "A" RESET "BC", ElideMiddle(input, 9)); +} + +#undef RESET +#undef RED +#undef NOTHING +#undef MAGENTA diff --git a/src/eval_env.cc b/src/eval_env.cc index e9b6c437e3..9f6a5dd9de 100644 --- a/src/eval_env.cc +++ b/src/eval_env.cc @@ -16,6 +16,8 @@ #include "eval_env.h" +using namespace std; + string BindingEnv::LookupVariable(const string& var) { map::iterator i = bindings_.find(var); if (i != bindings_.end()) @@ -29,22 +31,22 @@ void BindingEnv::AddBinding(const string& key, const string& val) { bindings_[key] = val; } -void BindingEnv::AddRule(const Rule* rule) { +void BindingEnv::AddRule(std::unique_ptr rule) { assert(LookupRuleCurrentScope(rule->name()) == NULL); - rules_[rule->name()] = rule; + rules_[rule->name()] = std::move(rule); } const Rule* BindingEnv::LookupRuleCurrentScope(const string& rule_name) { - map::iterator i = rules_.find(rule_name); + auto i = rules_.find(rule_name); if (i == rules_.end()) return NULL; - return i->second; + return i->second.get(); } const Rule* BindingEnv::LookupRule(const string& rule_name) { - map::iterator i = rules_.find(rule_name); + auto i = rules_.find(rule_name); if (i != rules_.end()) - return i->second; + return i->second.get(); if (parent_) return parent_->LookupRule(rule_name); return NULL; @@ -61,6 +63,16 @@ const EvalString* Rule::GetBinding(const string& key) const { return &i->second; } +std::unique_ptr Rule::Phony() { + auto rule = std::unique_ptr(new Rule("phony")); + rule->phony_ = true; + return rule; +} + +bool Rule::IsPhony() const { + return phony_; +} + // static bool Rule::IsReservedBinding(const string& var) { return var == "command" || @@ -76,7 +88,7 @@ bool Rule::IsReservedBinding(const string& var) { var == "msvc_deps_prefix"; } -const map& BindingEnv::GetRules() const { +const map>& BindingEnv::GetRules() const { return rules_; } @@ -97,6 +109,10 @@ string BindingEnv::LookupWithFallback(const string& var, } string EvalString::Evaluate(Env* env) const { + if (parsed_.empty()) { + return single_token_; + } + string result; for (TokenList::const_iterator i = parsed_.begin(); i != parsed_.end(); ++i) { if (i->second == RAW) @@ -108,40 +124,57 @@ string EvalString::Evaluate(Env* env) const { } void EvalString::AddText(StringPiece text) { - // Add it to the end of an existing RAW token if possible. - if (!parsed_.empty() && parsed_.back().second == RAW) { - parsed_.back().first.append(text.str_, text.len_); + if (parsed_.empty()) { + single_token_.append(text.begin(), text.end()); + } else if (!parsed_.empty() && parsed_.back().second == RAW) { + parsed_.back().first.append(text.begin(), text.end()); } else { - parsed_.push_back(make_pair(text.AsString(), RAW)); + parsed_.push_back(std::make_pair(text.AsString(), RAW)); } } + void EvalString::AddSpecial(StringPiece text) { - parsed_.push_back(make_pair(text.AsString(), SPECIAL)); + if (parsed_.empty() && !single_token_.empty()) { + // Going from one to two tokens, so we can no longer apply + // our single_token_ optimization and need to push everything + // onto the vector. + parsed_.push_back(std::make_pair(std::move(single_token_), RAW)); + } + parsed_.push_back(std::make_pair(text.AsString(), SPECIAL)); } string EvalString::Serialize() const { string result; - for (TokenList::const_iterator i = parsed_.begin(); - i != parsed_.end(); ++i) { + if (parsed_.empty() && !single_token_.empty()) { result.append("["); - if (i->second == SPECIAL) - result.append("$"); - result.append(i->first); + result.append(single_token_); result.append("]"); + } else { + for (const auto& pair : parsed_) { + result.append("["); + if (pair.second == SPECIAL) + result.append("$"); + result.append(pair.first.begin(), pair.first.end()); + result.append("]"); + } } return result; } string EvalString::Unparse() const { string result; - for (TokenList::const_iterator i = parsed_.begin(); - i != parsed_.end(); ++i) { - bool special = (i->second == SPECIAL); - if (special) - result.append("${"); - result.append(i->first); - if (special) - result.append("}"); + if (parsed_.empty() && !single_token_.empty()) { + result.append(single_token_.begin(), single_token_.end()); + } else { + for (TokenList::const_iterator i = parsed_.begin(); + i != parsed_.end(); ++i) { + bool special = (i->second == SPECIAL); + if (special) + result.append("${"); + result.append(i->first.begin(), i->first.end()); + if (special) + result.append("}"); + } } return result; } diff --git a/src/eval_env.h b/src/eval_env.h index 8fb9bf4e33..84c87fcbca 100644 --- a/src/eval_env.h +++ b/src/eval_env.h @@ -16,9 +16,9 @@ #define NINJA_EVAL_ENV_H_ #include +#include #include #include -using namespace std; #include "string_piece.h" @@ -27,7 +27,7 @@ struct Rule; /// An interface for a scope for variable (e.g. "$foo") lookups. struct Env { virtual ~Env() {} - virtual string LookupVariable(const string& var) = 0; + virtual std::string LookupVariable(const std::string& var) = 0; }; /// A tokenized string that contains variable references. @@ -35,46 +35,57 @@ struct Env { struct EvalString { /// @return The evaluated string with variable expanded using value found in /// environment @a env. - string Evaluate(Env* env) const; + std::string Evaluate(Env* env) const; /// @return The string with variables not expanded. - string Unparse() const; + std::string Unparse() const; - void Clear() { parsed_.clear(); } - bool empty() const { return parsed_.empty(); } + void Clear() { parsed_.clear(); single_token_.clear(); } + bool empty() const { return parsed_.empty() && single_token_.empty(); } void AddText(StringPiece text); void AddSpecial(StringPiece text); /// Construct a human-readable representation of the parsed state, /// for use in tests. - string Serialize() const; + std::string Serialize() const; private: enum TokenType { RAW, SPECIAL }; - typedef vector > TokenList; + typedef std::vector > TokenList; TokenList parsed_; + + // If we hold only a single RAW token, then we keep it here instead of + // pushing it on TokenList. This saves a bunch of allocations for + // what is a common case. If parsed_ is nonempty, then this value + // must be ignored. + std::string single_token_; }; -/// An invokable build command and associated metadata (description, etc.). +/// An invocable build command and associated metadata (description, etc.). struct Rule { - explicit Rule(const string& name) : name_(name) {} + explicit Rule(const std::string& name) : name_(name) {} + + static std::unique_ptr Phony(); + + bool IsPhony() const; - const string& name() const { return name_; } + const std::string& name() const { return name_; } - void AddBinding(const string& key, const EvalString& val); + void AddBinding(const std::string& key, const EvalString& val); - static bool IsReservedBinding(const string& var); + static bool IsReservedBinding(const std::string& var); - const EvalString* GetBinding(const string& key) const; + const EvalString* GetBinding(const std::string& key) const; private: // Allow the parsers to reach into this object and fill out its fields. friend struct ManifestParser; - string name_; - typedef map Bindings; + std::string name_; + typedef std::map Bindings; Bindings bindings_; + bool phony_ = false; }; /// An Env which contains a mapping of variables to values @@ -84,26 +95,26 @@ struct BindingEnv : public Env { explicit BindingEnv(BindingEnv* parent) : parent_(parent) {} virtual ~BindingEnv() {} - virtual string LookupVariable(const string& var); + virtual std::string LookupVariable(const std::string& var); - void AddRule(const Rule* rule); - const Rule* LookupRule(const string& rule_name); - const Rule* LookupRuleCurrentScope(const string& rule_name); - const map& GetRules() const; + void AddRule(std::unique_ptr rule); + const Rule* LookupRule(const std::string& rule_name); + const Rule* LookupRuleCurrentScope(const std::string& rule_name); + const std::map>& GetRules() const; - void AddBinding(const string& key, const string& val); + void AddBinding(const std::string& key, const std::string& val); /// This is tricky. Edges want lookup scope to go in this order: /// 1) value set on edge itself (edge_->env_) /// 2) value set on rule, with expansion in the edge's scope /// 3) value set on enclosing scope of edge (edge_->env_->parent_) /// This function takes as parameters the necessary info to do (2). - string LookupWithFallback(const string& var, const EvalString* eval, - Env* env); + std::string LookupWithFallback(const std::string& var, const EvalString* eval, + Env* env); private: - map bindings_; - map rules_; + std::map bindings_; + std::map> rules_; BindingEnv* parent_; }; diff --git a/src/exit_status.h b/src/exit_status.h index a714ece791..73f9470b46 100644 --- a/src/exit_status.h +++ b/src/exit_status.h @@ -15,10 +15,19 @@ #ifndef NINJA_EXIT_STATUS_H_ #define NINJA_EXIT_STATUS_H_ -enum ExitStatus { - ExitSuccess, +// The underlying type of the ExitStatus enum, used to represent a platform-specific +// process exit code. +#ifdef _WIN32 +#define EXIT_STATUS_TYPE unsigned long +#else // !_WIN32 +#define EXIT_STATUS_TYPE int +#endif // !_WIN32 + + +enum ExitStatus : EXIT_STATUS_TYPE { + ExitSuccess=0, ExitFailure, - ExitInterrupted + ExitInterrupted=130, }; #endif // NINJA_EXIT_STATUS_H_ diff --git a/src/explanations.h b/src/explanations.h new file mode 100644 index 0000000000..375b29f283 --- /dev/null +++ b/src/explanations.h @@ -0,0 +1,88 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +#include +#include +#include + +/// A class used to record a list of explanation strings associated +/// with a given 'item' pointer. This is used to implement the +/// `-d explain` feature. +struct Explanations { + public: + /// Record an explanation for |item| if this instance is enabled. + void Record(const void* item, const char* fmt, ...) { + va_list args; + va_start(args, fmt); + RecordArgs(item, fmt, args); + va_end(args); + } + + /// Same as Record(), but uses a va_list to pass formatting arguments. + void RecordArgs(const void* item, const char* fmt, va_list args) { + char buffer[1024]; + vsnprintf(buffer, sizeof(buffer), fmt, args); + map_[item].emplace_back(buffer); + } + + /// Lookup the explanations recorded for |item|, and append them + /// to |*out|, if any. + void LookupAndAppend(const void* item, std::vector* out) { + auto it = map_.find(item); + if (it == map_.end()) + return; + + for (const auto& explanation : it->second) + out->push_back(explanation); + } + + private: + std::unordered_map> map_; +}; + +/// Convenience wrapper for an Explanations pointer, which can be null +/// if no explanations need to be recorded. +struct OptionalExplanations { + OptionalExplanations(Explanations* explanations) + : explanations_(explanations) {} + + void Record(const void* item, const char* fmt, ...) { + if (explanations_) { + va_list args; + va_start(args, fmt); + explanations_->RecordArgs(item, fmt, args); + va_end(args); + } + } + + void RecordArgs(const void* item, const char* fmt, va_list args) { + if (explanations_) + explanations_->RecordArgs(item, fmt, args); + } + + void LookupAndAppend(const void* item, std::vector* out) { + if (explanations_) + explanations_->LookupAndAppend(item, out); + } + + Explanations* ptr() const { return explanations_; } + + private: + Explanations* explanations_; +}; diff --git a/src/explanations_test.cc b/src/explanations_test.cc new file mode 100644 index 0000000000..e46600f7a3 --- /dev/null +++ b/src/explanations_test.cc @@ -0,0 +1,97 @@ +// Copyright 2024 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "explanations.h" + +#include "test.h" + +namespace { + +const void* MakeItem(size_t v) { + return reinterpret_cast(v); +} + +} // namespace + +TEST(Explanations, Explanations) { + Explanations exp; + + exp.Record(MakeItem(1), "first explanation"); + exp.Record(MakeItem(1), "second explanation"); + exp.Record(MakeItem(2), "third explanation"); + exp.Record(MakeItem(2), "fourth %s", "explanation"); + + std::vector list; + + exp.LookupAndAppend(MakeItem(0), &list); + ASSERT_TRUE(list.empty()); + + exp.LookupAndAppend(MakeItem(1), &list); + ASSERT_EQ(2u, list.size()); + EXPECT_EQ(list[0], "first explanation"); + EXPECT_EQ(list[1], "second explanation"); + + exp.LookupAndAppend(MakeItem(2), &list); + ASSERT_EQ(4u, list.size()); + EXPECT_EQ(list[0], "first explanation"); + EXPECT_EQ(list[1], "second explanation"); + EXPECT_EQ(list[2], "third explanation"); + EXPECT_EQ(list[3], "fourth explanation"); +} + +TEST(Explanations, OptionalExplanationsNonNull) { + Explanations parent; + OptionalExplanations exp(&parent); + + exp.Record(MakeItem(1), "first explanation"); + exp.Record(MakeItem(1), "second explanation"); + exp.Record(MakeItem(2), "third explanation"); + exp.Record(MakeItem(2), "fourth %s", "explanation"); + + std::vector list; + + exp.LookupAndAppend(MakeItem(0), &list); + ASSERT_TRUE(list.empty()); + + exp.LookupAndAppend(MakeItem(1), &list); + ASSERT_EQ(2u, list.size()); + EXPECT_EQ(list[0], "first explanation"); + EXPECT_EQ(list[1], "second explanation"); + + exp.LookupAndAppend(MakeItem(2), &list); + ASSERT_EQ(4u, list.size()); + EXPECT_EQ(list[0], "first explanation"); + EXPECT_EQ(list[1], "second explanation"); + EXPECT_EQ(list[2], "third explanation"); + EXPECT_EQ(list[3], "fourth explanation"); +} + +TEST(Explanations, OptionalExplanationsWithNullPointer) { + OptionalExplanations exp(nullptr); + + exp.Record(MakeItem(1), "first explanation"); + exp.Record(MakeItem(1), "second explanation"); + exp.Record(MakeItem(2), "third explanation"); + exp.Record(MakeItem(2), "fourth %s", "explanation"); + + std::vector list; + exp.LookupAndAppend(MakeItem(0), &list); + ASSERT_TRUE(list.empty()); + + exp.LookupAndAppend(MakeItem(1), &list); + ASSERT_TRUE(list.empty()); + + exp.LookupAndAppend(MakeItem(2), &list); + ASSERT_TRUE(list.empty()); +} diff --git a/src/graph.cc b/src/graph.cc index a90c049e6e..84c600615a 100644 --- a/src/graph.cc +++ b/src/graph.cc @@ -14,6 +14,8 @@ #include "graph.h" +#include +#include #include #include @@ -27,17 +29,59 @@ #include "state.h" #include "util.h" +using namespace std; + bool Node::Stat(DiskInterface* disk_interface, string* err) { - return (mtime_ = disk_interface->Stat(path_, err)) != -1; + mtime_ = disk_interface->Stat(path_, err); + if (mtime_ == -1) { + return false; + } + exists_ = (mtime_ != 0) ? ExistenceStatusExists : ExistenceStatusMissing; + return true; } -bool DependencyScan::RecomputeDirty(Node* node, string* err) { - vector stack; - return RecomputeDirty(node, &stack, err); +void Node::UpdatePhonyMtime(TimeStamp mtime) { + if (!exists()) { + mtime_ = std::max(mtime_, mtime); + } } -bool DependencyScan::RecomputeDirty(Node* node, vector* stack, +bool DependencyScan::RecomputeDirty(Node* initial_node, + std::vector* validation_nodes, string* err) { + std::vector stack; + std::vector new_validation_nodes; + + std::deque nodes(1, initial_node); + + // RecomputeNodeDirty might return new validation nodes that need to be + // checked for dirty state, keep a queue of nodes to visit. + while (!nodes.empty()) { + Node* node = nodes.front(); + nodes.pop_front(); + + stack.clear(); + new_validation_nodes.clear(); + + if (!RecomputeNodeDirty(node, &stack, &new_validation_nodes, err)) + return false; + nodes.insert(nodes.end(), new_validation_nodes.begin(), + new_validation_nodes.end()); + if (!new_validation_nodes.empty()) { + assert(validation_nodes && + "validations require RecomputeDirty to be called with validation_nodes"); + validation_nodes->insert(validation_nodes->end(), + new_validation_nodes.begin(), + new_validation_nodes.end()); + } + } + + return true; +} + +bool DependencyScan::RecomputeNodeDirty(Node* node, std::vector* stack, + std::vector* validation_nodes, + string* err) { Edge* edge = node->in_edge(); if (!edge) { // If we already visited this leaf node then we are done. @@ -47,7 +91,8 @@ bool DependencyScan::RecomputeDirty(Node* node, vector* stack, if (!node->StatIfNecessary(disk_interface_, err)) return false; if (!node->exists()) - EXPLAIN("%s has no in-edge and is missing", node->path().c_str()); + explanations_.Record(node, "%s has no in-edge and is missing", + node->path().c_str()); node->set_dirty(!node->exists()); return true; } @@ -81,7 +126,7 @@ bool DependencyScan::RecomputeDirty(Node* node, vector* stack, // Later during the build the dyndep file will become ready and be // loaded to update this edge before it can possibly be scheduled. if (edge->dyndep_ && edge->dyndep_->dyndep_pending()) { - if (!RecomputeDirty(edge->dyndep_, stack, err)) + if (!RecomputeNodeDirty(edge->dyndep_, stack, validation_nodes, err)) return false; if (!edge->dyndep_->in_edge() || @@ -107,17 +152,25 @@ bool DependencyScan::RecomputeDirty(Node* node, vector* stack, if (!err->empty()) return false; // Failed to load dependency info: rebuild to regenerate it. - // LoadDeps() did EXPLAIN() already, no need to do it here. + // LoadDeps() did explanations_->Record() already, no need to do it here. dirty = edge->deps_missing_ = true; } } + // Store any validation nodes from the edge for adding to the initial + // nodes. Don't recurse into them, that would trigger the dependency + // cycle detector if the validation node depends on this node. + // RecomputeDirty will add the validation nodes to the initial nodes + // and recurse into them. + validation_nodes->insert(validation_nodes->end(), + edge->validations_.begin(), edge->validations_.end()); + // Visit all inputs; we're dirty if any of the inputs are dirty. Node* most_recent_input = NULL; for (vector::iterator i = edge->inputs_.begin(); i != edge->inputs_.end(); ++i) { // Visit this input. - if (!RecomputeDirty(*i, stack, err)) + if (!RecomputeNodeDirty(*i, stack, validation_nodes, err)) return false; // If an input is not ready, neither are our outputs. @@ -130,7 +183,7 @@ bool DependencyScan::RecomputeDirty(Node* node, vector* stack, // If a regular input is dirty (or missing), we're dirty. // Otherwise consider mtime. if ((*i)->dirty()) { - EXPLAIN("%s is dirty", (*i)->path().c_str()); + explanations_.Record(node, "%s is dirty", (*i)->path().c_str()); dirty = true; } else { if (!most_recent_input || (*i)->mtime() > most_recent_input->mtime()) { @@ -222,52 +275,60 @@ bool DependencyScan::RecomputeOutputsDirty(Edge* edge, Node* most_recent_input, return true; } -bool DependencyScan::RecomputeOutputDirty(Edge* edge, - Node* most_recent_input, +bool DependencyScan::RecomputeOutputDirty(const Edge* edge, + const Node* most_recent_input, const string& command, Node* output) { if (edge->is_phony()) { // Phony edges don't write any output. Outputs are only dirty if // there are no inputs and we're missing the output. if (edge->inputs_.empty() && !output->exists()) { - EXPLAIN("output %s of phony edge with no inputs doesn't exist", - output->path().c_str()); + explanations_.Record( + output, "output %s of phony edge with no inputs doesn't exist", + output->path().c_str()); return true; } + + // Update the mtime with the newest input. Dependents can thus call mtime() + // on the fake node and get the latest mtime of the dependencies + if (most_recent_input) { + output->UpdatePhonyMtime(most_recent_input->mtime()); + } + + // Phony edges are clean, nothing to do return false; } - BuildLog::LogEntry* entry = 0; - // Dirty if we're missing the output. if (!output->exists()) { - EXPLAIN("output %s doesn't exist", output->path().c_str()); + explanations_.Record(output, "output %s doesn't exist", + output->path().c_str()); return true; } - // Dirty if the output is older than the input. - if (most_recent_input && output->mtime() < most_recent_input->mtime()) { - TimeStamp output_mtime = output->mtime(); - - // If this is a restat rule, we may have cleaned the output with a restat - // rule in a previous run and stored the most recent input mtime in the - // build log. Use that mtime instead, so that the file will only be - // considered dirty if an input was modified since the previous run. - bool used_restat = false; - if (edge->GetBindingBool("restat") && build_log() && - (entry = build_log()->LookupByOutput(output->path()))) { - output_mtime = entry->mtime; - used_restat = true; - } + BuildLog::LogEntry* entry = 0; - if (output_mtime < most_recent_input->mtime()) { - EXPLAIN("%soutput %s older than most recent input %s " - "(%" PRId64 " vs %" PRId64 ")", - used_restat ? "restat of " : "", output->path().c_str(), - most_recent_input->path().c_str(), - output_mtime, most_recent_input->mtime()); - return true; - } + // If this is a restat rule, we may have cleaned the output in a + // previous run and stored the command start time in the build log. + // We don't want to consider a restat rule's outputs as dirty unless + // an input changed since the last run, so we'll skip checking the + // output file's actual mtime and simply check the recorded mtime from + // the log against the most recent input's mtime (see below) + bool used_restat = false; + if (edge->GetBindingBool("restat") && build_log() && + (entry = build_log()->LookupByOutput(output->path()))) { + used_restat = true; + } + + // Dirty if the output is older than the input. + if (!used_restat && most_recent_input && output->mtime() < most_recent_input->mtime()) { + explanations_.Record(output, + "output %s older than most recent input %s " + "(%" PRId64 " vs %" PRId64 ")", + output->path().c_str(), + most_recent_input->path().c_str(), output->mtime(), + most_recent_input->mtime()); + return true; } if (build_log()) { @@ -278,22 +339,29 @@ bool DependencyScan::RecomputeOutputDirty(Edge* edge, // May also be dirty due to the command changing since the last build. // But if this is a generator rule, the command changing does not make us // dirty. - EXPLAIN("command line changed for %s", output->path().c_str()); + explanations_.Record(output, "command line changed for %s", + output->path().c_str()); return true; } if (most_recent_input && entry->mtime < most_recent_input->mtime()) { // May also be dirty due to the mtime in the log being older than the // mtime of the most recent input. This can occur even when the mtime // on disk is newer if a previous run wrote to the output file but - // exited with an error or was interrupted. - EXPLAIN("recorded mtime of %s older than most recent input %s (%" PRId64 " vs %" PRId64 ")", - output->path().c_str(), most_recent_input->path().c_str(), - entry->mtime, most_recent_input->mtime()); + // exited with an error or was interrupted. If this was a restat rule, + // then we only check the recorded mtime against the most recent input + // mtime and ignore the actual output's mtime above. + explanations_.Record( + output, + "recorded mtime of %s older than most recent input %s (%" PRId64 + " vs %" PRId64 ")", + output->path().c_str(), most_recent_input->path().c_str(), + entry->mtime, most_recent_input->mtime()); return true; } } if (!entry && !generator) { - EXPLAIN("command line not found in log for %s", output->path().c_str()); + explanations_.Record(output, "command line not found in log for %s", + output->path().c_str()); return true; } } @@ -323,19 +391,17 @@ bool Edge::AllInputsReady() const { struct EdgeEnv : public Env { enum EscapeKind { kShellEscape, kDoNotEscape }; - EdgeEnv(Edge* edge, EscapeKind escape) + EdgeEnv(const Edge* const edge, const EscapeKind escape) : edge_(edge), escape_in_out_(escape), recursive_(false) {} virtual string LookupVariable(const string& var); /// Given a span of Nodes, construct a list of paths suitable for a command /// line. - string MakePathList(vector::iterator begin, - vector::iterator end, - char sep); + std::string MakePathList(const Node* const* span, size_t size, char sep) const; private: - vector lookups_; - Edge* edge_; + std::vector lookups_; + const Edge* const edge_; EscapeKind escape_in_out_; bool recursive_; }; @@ -344,20 +410,50 @@ string EdgeEnv::LookupVariable(const string& var) { if (var == "in" || var == "in_newline") { int explicit_deps_count = edge_->inputs_.size() - edge_->implicit_deps_ - edge_->order_only_deps_; - return MakePathList(edge_->inputs_.begin(), - edge_->inputs_.begin() + explicit_deps_count, + return MakePathList(edge_->inputs_.data(), explicit_deps_count, var == "in" ? ' ' : '\n'); } else if (var == "out") { int explicit_outs_count = edge_->outputs_.size() - edge_->implicit_outs_; - return MakePathList(edge_->outputs_.begin(), - edge_->outputs_.begin() + explicit_outs_count, - ' '); - } - + return MakePathList(&edge_->outputs_[0], explicit_outs_count, ' '); + } + + // Technical note about the lookups_ vector. + // + // This is used to detect cycles during recursive variable expansion + // which can be seen as a graph traversal problem. Consider the following + // example: + // + // rule something + // command = $foo $foo $var1 + // var1 = $var2 + // var2 = $var3 + // var3 = $var1 + // foo = FOO + // + // Each variable definition can be seen as a node in a graph that looks + // like the following: + // + // command --> foo + // | + // v + // var1 <-----. + // | | + // v | + // var2 ---> var3 + // + // The lookups_ vector is used as a stack of visited nodes/variables + // during recursive expansion. Entering a node adds an item to the + // stack, leaving the node removes it. + // + // The recursive_ flag is used as a small performance optimization + // to never record the starting node in the stack when beginning a new + // expansion, since in most cases, expansions are not recursive + // at all. + // if (recursive_) { - vector::const_iterator it; - if ((it = find(lookups_.begin(), lookups_.end(), var)) != lookups_.end()) { - string cycle; + auto it = std::find(lookups_.begin(), lookups_.end(), var); + if (it != lookups_.end()) { + std::string cycle; for (; it != lookups_.end(); ++it) cycle.append(*it + " -> "); cycle.append(var); @@ -367,25 +463,28 @@ string EdgeEnv::LookupVariable(const string& var) { // See notes on BindingEnv::LookupWithFallback. const EvalString* eval = edge_->rule_->GetBinding(var); - if (recursive_ && eval) + bool record_varname = recursive_ && eval; + if (record_varname) lookups_.push_back(var); // In practice, variables defined on rules never use another rule variable. // For performance, only start checking for cycles after the first lookup. recursive_ = true; - return edge_->env_->LookupWithFallback(var, eval, this); + std::string result = edge_->env_->LookupWithFallback(var, eval, this); + if (record_varname) + lookups_.pop_back(); + return result; } -string EdgeEnv::MakePathList(vector::iterator begin, - vector::iterator end, - char sep) { +std::string EdgeEnv::MakePathList(const Node* const* const span, + const size_t size, const char sep) const { string result; - for (vector::iterator i = begin; i != end; ++i) { + for (const Node* const* i = span; i != span + size; ++i) { if (!result.empty()) result.push_back(sep); const string& path = (*i)->PathDecanonicalized(); if (escape_in_out_ == kShellEscape) { -#if _WIN32 +#ifdef _WIN32 GetWin32EscapedString(path, &result); #else GetShellEscapedString(path, &result); @@ -397,7 +496,7 @@ string EdgeEnv::MakePathList(vector::iterator begin, return result; } -string Edge::EvaluateCommand(bool incl_rsp_file) { +std::string Edge::EvaluateCommand(const bool incl_rsp_file) const { string command = GetBinding("command"); if (incl_rsp_file) { string rspfile_content = GetBinding("rspfile_content"); @@ -407,26 +506,26 @@ string Edge::EvaluateCommand(bool incl_rsp_file) { return command; } -string Edge::GetBinding(const string& key) { +std::string Edge::GetBinding(const std::string& key) const { EdgeEnv env(this, EdgeEnv::kShellEscape); return env.LookupVariable(key); } -bool Edge::GetBindingBool(const string& key) { +bool Edge::GetBindingBool(const string& key) const { return !GetBinding(key).empty(); } -string Edge::GetUnescapedDepfile() { +string Edge::GetUnescapedDepfile() const { EdgeEnv env(this, EdgeEnv::kDoNotEscape); return env.LookupVariable("depfile"); } -string Edge::GetUnescapedDyndep() { +string Edge::GetUnescapedDyndep() const { EdgeEnv env(this, EdgeEnv::kDoNotEscape); return env.LookupVariable("dyndep"); } -string Edge::GetUnescapedRspfile() { +std::string Edge::GetUnescapedRspfile() const { EdgeEnv env(this, EdgeEnv::kDoNotEscape); return env.LookupVariable("rspfile"); } @@ -442,6 +541,13 @@ void Edge::Dump(const char* prefix) const { i != outputs_.end() && *i != NULL; ++i) { printf("%s ", (*i)->path().c_str()); } + if (!validations_.empty()) { + printf(" validations "); + for (std::vector::const_iterator i = validations_.begin(); + i != validations_.end() && *i != NULL; ++i) { + printf("%s ", (*i)->path().c_str()); + } + } if (pool_) { if (!pool_->name().empty()) { printf("(in pool '%s')", pool_->name().c_str()); @@ -453,7 +559,7 @@ void Edge::Dump(const char* prefix) const { } bool Edge::is_phony() const { - return rule_ == &State::kPhonyRule; + return rule_->IsPhony(); } bool Edge::use_console() const { @@ -486,7 +592,7 @@ string Node::PathDecanonicalized(const string& path, uint64_t slash_bits) { void Node::Dump(const char* prefix) const { printf("%s <%s 0x%p> mtime: %" PRId64 "%s, (:%s), ", prefix, path().c_str(), this, - mtime(), mtime() ? "" : " (:missing)", + mtime(), exists() ? "" : " (:missing)", dirty() ? " dirty" : " clean"); if (in_edge()) { in_edge()->Dump("in-edge: "); @@ -498,6 +604,13 @@ void Node::Dump(const char* prefix) const { e != out_edges().end() && *e != NULL; ++e) { (*e)->Dump(" +- "); } + if (!validation_out_edges().empty()) { + printf(" validation out edges:\n"); + for (std::vector::const_iterator e = validation_out_edges().begin(); + e != validation_out_edges().end() && *e != NULL; ++e) { + (*e)->Dump(" +- "); + } + } } bool ImplicitDepLoader::LoadDeps(Edge* edge, string* err) { @@ -513,6 +626,17 @@ bool ImplicitDepLoader::LoadDeps(Edge* edge, string* err) { return true; } +struct matches { + explicit matches(std::vector::iterator i) : i_(i) {} + + bool operator()(const Node* node) const { + StringPiece opath = StringPiece(node->path()); + return *i_ == opath; + } + + std::vector::iterator i_; +}; + bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, string* err) { METRIC_RECORD("depfile load"); @@ -529,8 +653,9 @@ bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, return false; } // On a missing depfile: return false and empty *err. + Node* first_output = edge->outputs_[0]; if (content.empty()) { - EXPLAIN("depfile '%s' is missing", path.c_str()); + explanations_.Record(first_output, "depfile '%s' is missing", path.c_str()); return false; } @@ -543,39 +668,54 @@ bool ImplicitDepLoader::LoadDepFile(Edge* edge, const string& path, return false; } - uint64_t unused; - if (!CanonicalizePath(const_cast(depfile.out_.str_), - &depfile.out_.len_, &unused, err)) { - *err = path + ": " + *err; + if (depfile.outs_.empty()) { + *err = path + ": no outputs declared"; return false; } + uint64_t unused; + std::vector::iterator primary_out = depfile.outs_.begin(); + CanonicalizePath(const_cast(primary_out->str_), &primary_out->len_, + &unused); + // Check that this depfile matches the edge's output, if not return false to // mark the edge as dirty. - Node* first_output = edge->outputs_[0]; StringPiece opath = StringPiece(first_output->path()); - if (opath != depfile.out_) { - EXPLAIN("expected depfile '%s' to mention '%s', got '%s'", path.c_str(), - first_output->path().c_str(), depfile.out_.AsString().c_str()); + if (opath != *primary_out) { + explanations_.Record(first_output, + "expected depfile '%s' to mention '%s', got '%s'", + path.c_str(), first_output->path().c_str(), + primary_out->AsString().c_str()); return false; } + // Ensure that all mentioned outputs are outputs of the edge. + for (std::vector::iterator o = depfile.outs_.begin(); + o != depfile.outs_.end(); ++o) { + matches m(o); + if (std::find_if(edge->outputs_.begin(), edge->outputs_.end(), m) == edge->outputs_.end()) { + *err = path + ": depfile mentions '" + o->AsString() + "' as an output, but no such output was declared"; + return false; + } + } + + return ProcessDepfileDeps(edge, &depfile.ins_, err); +} + +bool ImplicitDepLoader::ProcessDepfileDeps( + Edge* edge, std::vector* depfile_ins, std::string* err) { // Preallocate space in edge->inputs_ to be filled in below. vector::iterator implicit_dep = - PreallocateSpace(edge, depfile.ins_.size()); + PreallocateSpace(edge, depfile_ins->size()); // Add all its in-edges. - for (vector::iterator i = depfile.ins_.begin(); - i != depfile.ins_.end(); ++i, ++implicit_dep) { + for (std::vector::iterator i = depfile_ins->begin(); + i != depfile_ins->end(); ++i, ++implicit_dep) { uint64_t slash_bits; - if (!CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits, - err)) - return false; - + CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits); Node* node = state_->GetNode(*i, slash_bits); *implicit_dep = node; node->AddOutEdge(edge); - CreatePhonyInEdge(node); } return true; @@ -586,24 +726,27 @@ bool ImplicitDepLoader::LoadDepsFromLog(Edge* edge, string* err) { Node* output = edge->outputs_[0]; DepsLog::Deps* deps = deps_log_ ? deps_log_->GetDeps(output) : NULL; if (!deps) { - EXPLAIN("deps for '%s' are missing", output->path().c_str()); + explanations_.Record(output, "deps for '%s' are missing", + output->path().c_str()); return false; } // Deps are invalid if the output is newer than the deps. if (output->mtime() > deps->mtime) { - EXPLAIN("stored deps info out of date for '%s' (%" PRId64 " vs %" PRId64 ")", - output->path().c_str(), deps->mtime, output->mtime()); + explanations_.Record(output, + "stored deps info out of date for '%s' (%" PRId64 + " vs %" PRId64 ")", + output->path().c_str(), deps->mtime, output->mtime()); return false; } - vector::iterator implicit_dep = - PreallocateSpace(edge, deps->node_count); - for (int i = 0; i < deps->node_count; ++i, ++implicit_dep) { - Node* node = deps->nodes[i]; - *implicit_dep = node; - node->AddOutEdge(edge); - CreatePhonyInEdge(node); + Node** nodes = deps->nodes; + size_t node_count = deps->node_count; + edge->inputs_.insert(edge->inputs_.end() - edge->order_only_deps_, + nodes, nodes + node_count); + edge->implicit_deps_ += node_count; + for (size_t i = 0; i < node_count; ++i) { + nodes[i]->AddOutEdge(edge); } return true; } @@ -616,19 +759,46 @@ vector::iterator ImplicitDepLoader::PreallocateSpace(Edge* edge, return edge->inputs_.end() - edge->order_only_deps_ - count; } -void ImplicitDepLoader::CreatePhonyInEdge(Node* node) { - if (node->in_edge()) +void InputsCollector::VisitNode(const Node* node) { + const Edge* edge = node->in_edge(); + + if (!edge) // A source file. return; - Edge* phony_edge = state_->AddEdge(&State::kPhonyRule); - node->set_in_edge(phony_edge); - phony_edge->outputs_.push_back(node); - - // RecomputeDirty might not be called for phony_edge if a previous call - // to RecomputeDirty had caused the file to be stat'ed. Because previous - // invocations of RecomputeDirty would have seen this node without an - // input edge (and therefore ready), we have to set outputs_ready_ to true - // to avoid a potential stuck build. If we do call RecomputeDirty for - // this node, it will simply set outputs_ready_ to the correct value. - phony_edge->outputs_ready_ = true; + // Add inputs of the producing edge to the result, + // except if they are themselves produced by a phony + // edge. + for (const Node* input : edge->inputs_) { + if (!visited_nodes_.insert(input).second) + continue; + + VisitNode(input); + + const Edge* input_edge = input->in_edge(); + if (!(input_edge && input_edge->is_phony())) { + inputs_.push_back(input); + } + } +} + +std::vector InputsCollector::GetInputsAsStrings( + bool shell_escape) const { + std::vector result; + result.reserve(inputs_.size()); + + for (const Node* input : inputs_) { + std::string unescaped = input->PathDecanonicalized(); + if (shell_escape) { + std::string path; +#ifdef _WIN32 + GetWin32EscapedString(unescaped, &path); +#else + GetShellEscapedString(unescaped, &path); +#endif + result.push_back(std::move(path)); + } else { + result.push_back(std::move(unescaped)); + } + } + return result; } diff --git a/src/graph.h b/src/graph.h index 75edbc5cb9..806260e5d7 100644 --- a/src/graph.h +++ b/src/graph.h @@ -15,12 +15,15 @@ #ifndef NINJA_GRAPH_H_ #define NINJA_GRAPH_H_ +#include +#include +#include #include #include -using namespace std; #include "dyndep.h" #include "eval_env.h" +#include "explanations.h" #include "timestamp.h" #include "util.h" @@ -36,20 +39,17 @@ struct State; /// Information about a node in the dependency graph: the file, whether /// it's dirty, mtime, etc. struct Node { - Node(const string& path, uint64_t slash_bits) - : path_(path), - slash_bits_(slash_bits), - mtime_(-1), - dirty_(false), - dyndep_pending_(false), - in_edge_(NULL), - id_(-1) {} + Node(const std::string& path, uint64_t slash_bits) + : path_(path), slash_bits_(slash_bits) {} /// Return false on error. - bool Stat(DiskInterface* disk_interface, string* err); + bool Stat(DiskInterface* disk_interface, std::string* err); + + /// If the file doesn't exist, set the mtime_ from its dependencies + void UpdatePhonyMtime(TimeStamp mtime); /// Return false on error. - bool StatIfNecessary(DiskInterface* disk_interface, string* err) { + bool StatIfNecessary(DiskInterface* disk_interface, std::string* err) { if (status_known()) return true; return Stat(disk_interface, err); @@ -58,29 +58,33 @@ struct Node { /// Mark as not-yet-stat()ed and not dirty. void ResetState() { mtime_ = -1; + exists_ = ExistenceStatusUnknown; dirty_ = false; } /// Mark the Node as already-stat()ed and missing. void MarkMissing() { - mtime_ = 0; + if (mtime_ == -1) { + mtime_ = 0; + } + exists_ = ExistenceStatusMissing; } bool exists() const { - return mtime_ != 0; + return exists_ == ExistenceStatusExists; } bool status_known() const { - return mtime_ != -1; + return exists_ != ExistenceStatusUnknown; } - const string& path() const { return path_; } + const std::string& path() const { return path_; } /// Get |path()| but use slash_bits to convert back to original slash styles. - string PathDecanonicalized() const { + std::string PathDecanonicalized() const { return PathDecanonicalized(path_, slash_bits_); } - static string PathDecanonicalized(const string& path, - uint64_t slash_bits); + static std::string PathDecanonicalized(const std::string& path, + uint64_t slash_bits); uint64_t slash_bits() const { return slash_bits_; } TimeStamp mtime() const { return mtime_; } @@ -95,45 +99,75 @@ struct Node { Edge* in_edge() const { return in_edge_; } void set_in_edge(Edge* edge) { in_edge_ = edge; } + /// Indicates whether this node was generated from a depfile or dyndep file, + /// instead of being a regular input or output from the Ninja manifest. + bool generated_by_dep_loader() const { return generated_by_dep_loader_; } + + void set_generated_by_dep_loader(bool value) { + generated_by_dep_loader_ = value; + } + int id() const { return id_; } void set_id(int id) { id_ = id; } - const vector& out_edges() const { return out_edges_; } + const std::vector& out_edges() const { return out_edges_; } + const std::vector& validation_out_edges() const { return validation_out_edges_; } void AddOutEdge(Edge* edge) { out_edges_.push_back(edge); } + void AddValidationOutEdge(Edge* edge) { validation_out_edges_.push_back(edge); } void Dump(const char* prefix="") const; private: - string path_; + std::string path_; /// Set bits starting from lowest for backslashes that were normalized to /// forward slashes by CanonicalizePath. See |PathDecanonicalized|. - uint64_t slash_bits_; + uint64_t slash_bits_ = 0; /// Possible values of mtime_: /// -1: file hasn't been examined /// 0: we looked, and file doesn't exist - /// >0: actual file's mtime - TimeStamp mtime_; + /// >0: actual file's mtime, or the latest mtime of its dependencies if it doesn't exist + TimeStamp mtime_ = -1; + + enum ExistenceStatus { + /// The file hasn't been examined. + ExistenceStatusUnknown, + /// The file doesn't exist. mtime_ will be the latest mtime of its dependencies. + ExistenceStatusMissing, + /// The path is an actual file. mtime_ will be the file's mtime. + ExistenceStatusExists + }; + ExistenceStatus exists_ = ExistenceStatusUnknown; /// Dirty is true when the underlying file is out-of-date. /// But note that Edge::outputs_ready_ is also used in judging which /// edges to build. - bool dirty_; + bool dirty_ = false; /// Store whether dyndep information is expected from this node but /// has not yet been loaded. - bool dyndep_pending_; + bool dyndep_pending_ = false; + + /// Set to true when this node comes from a depfile, a dyndep file or the + /// deps log. If it does not have a producing edge, the build should not + /// abort if it is missing (as for regular source inputs). By default + /// all nodes have this flag set to true, since the deps and build logs + /// can be loaded before the manifest. + bool generated_by_dep_loader_ = true; /// The Edge that produces this Node, or NULL when there is no /// known edge to produce it. - Edge* in_edge_; + Edge* in_edge_ = nullptr; /// All Edges that use this Node as an input. - vector out_edges_; + std::vector out_edges_; + + /// All Edges that use this Node as a validation. + std::vector validation_out_edges_; /// A dense integer id for the node, assigned and used by DepsLog. - int id_; + int id_ = -1; }; /// An edge in the dependency graph; links between Nodes using Rules. @@ -144,10 +178,7 @@ struct Edge { VisitDone }; - Edge() : rule_(NULL), pool_(NULL), dyndep_(NULL), env_(NULL), - mark_(VisitNone), outputs_ready_(false), deps_loaded_(false), - deps_missing_(false), implicit_deps_(0), order_only_deps_(0), - implicit_outs_(0) {} + Edge() = default; /// Return true if all inputs' in-edges are ready. bool AllInputsReady() const; @@ -155,31 +186,45 @@ struct Edge { /// Expand all variables in a command and return it as a string. /// If incl_rsp_file is enabled, the string will also contain the /// full contents of a response file (if applicable) - string EvaluateCommand(bool incl_rsp_file = false); + std::string EvaluateCommand(bool incl_rsp_file = false) const; /// Returns the shell-escaped value of |key|. - string GetBinding(const string& key); - bool GetBindingBool(const string& key); + std::string GetBinding(const std::string& key) const; + bool GetBindingBool(const std::string& key) const; /// Like GetBinding("depfile"), but without shell escaping. - string GetUnescapedDepfile(); + std::string GetUnescapedDepfile() const; /// Like GetBinding("dyndep"), but without shell escaping. - string GetUnescapedDyndep(); + std::string GetUnescapedDyndep() const; /// Like GetBinding("rspfile"), but without shell escaping. - string GetUnescapedRspfile(); + std::string GetUnescapedRspfile() const; void Dump(const char* prefix="") const; - const Rule* rule_; - Pool* pool_; - vector inputs_; - vector outputs_; - Node* dyndep_; - BindingEnv* env_; - VisitMark mark_; - bool outputs_ready_; - bool deps_loaded_; - bool deps_missing_; + // critical_path_weight is the priority during build scheduling. The + // "critical path" between this edge's inputs and any target node is + // the path which maximises the sum oof weights along that path. + // NOTE: Defaults to -1 as a marker smaller than any valid weight + int64_t critical_path_weight() const { return critical_path_weight_; } + void set_critical_path_weight(int64_t critical_path_weight) { + critical_path_weight_ = critical_path_weight; + } + + const Rule* rule_ = nullptr; + Pool* pool_ = nullptr; + std::vector inputs_; + std::vector outputs_; + std::vector validations_; + Node* dyndep_ = nullptr; + BindingEnv* env_ = nullptr; + VisitMark mark_ = VisitNone; + size_t id_ = 0; + int64_t critical_path_weight_ = -1; + bool outputs_ready_ = false; + bool deps_loaded_ = false; + bool deps_missing_ = false; + bool generated_by_dep_loader_ = false; + TimeStamp command_start_time_ = 0; const Rule& rule() const { return *rule_; } Pool* pool() const { return pool_; } @@ -194,8 +239,8 @@ struct Edge { // don't cause the target to rebuild. // These are stored in inputs_ in that order, and we keep counts of // #2 and #3 when we need to access the various subsets. - int implicit_deps_; - int order_only_deps_; + int implicit_deps_ = 0; + int order_only_deps_ = 0; bool is_implicit(size_t index) { return index >= inputs_.size() - order_only_deps_ - implicit_deps_ && !is_order_only(index); @@ -209,7 +254,7 @@ struct Edge { // 2) implicit outs, which the target generates but are not part of $out. // These are stored in outputs_ in that order, and we keep a count of // #2 to use when we need to access the various subsets. - int implicit_outs_; + int implicit_outs_ = 0; bool is_implicit_out(size_t index) const { return index >= outputs_.size() - implicit_outs_; } @@ -217,49 +262,64 @@ struct Edge { bool is_phony() const; bool use_console() const; bool maybe_phonycycle_diagnostic() const; + + // Historical info: how long did this edge take last time, + // as per .ninja_log, if known? Defaults to -1 if unknown. + int64_t prev_elapsed_time_millis = -1; +}; + +struct EdgeCmp { + bool operator()(const Edge* a, const Edge* b) const { + return a->id_ < b->id_; + } }; +typedef std::set EdgeSet; /// ImplicitDepLoader loads implicit dependencies, as referenced via the /// "depfile" attribute in build files. struct ImplicitDepLoader { ImplicitDepLoader(State* state, DepsLog* deps_log, DiskInterface* disk_interface, - DepfileParserOptions const* depfile_parser_options) + DepfileParserOptions const* depfile_parser_options, + Explanations* explanations) : state_(state), disk_interface_(disk_interface), deps_log_(deps_log), - depfile_parser_options_(depfile_parser_options) {} + depfile_parser_options_(depfile_parser_options), + explanations_(explanations) {} /// Load implicit dependencies for \a edge. /// @return false on error (without filling \a err if info is just missing // or out of date). - bool LoadDeps(Edge* edge, string* err); + bool LoadDeps(Edge* edge, std::string* err); DepsLog* deps_log() const { return deps_log_; } - private: + protected: + /// Process loaded implicit dependencies for \a edge and update the graph + /// @return false on error (without filling \a err if info is just missing) + virtual bool ProcessDepfileDeps(Edge* edge, + std::vector* depfile_ins, + std::string* err); + /// Load implicit dependencies for \a edge from a depfile attribute. /// @return false on error (without filling \a err if info is just missing). - bool LoadDepFile(Edge* edge, const string& path, string* err); + bool LoadDepFile(Edge* edge, const std::string& path, std::string* err); /// Load implicit dependencies for \a edge from the DepsLog. /// @return false on error (without filling \a err if info is just missing). - bool LoadDepsFromLog(Edge* edge, string* err); + bool LoadDepsFromLog(Edge* edge, std::string* err); /// Preallocate \a count spaces in the input array on \a edge, returning /// an iterator pointing at the first new space. - vector::iterator PreallocateSpace(Edge* edge, int count); - - /// If we don't have a edge that generates this input already, - /// create one; this makes us not abort if the input is missing, - /// but instead will rebuild in that circumstance. - void CreatePhonyInEdge(Node* node); + std::vector::iterator PreallocateSpace(Edge* edge, int count); State* state_; DiskInterface* disk_interface_; DepsLog* deps_log_; DepfileParserOptions const* depfile_parser_options_; + OptionalExplanations explanations_; }; @@ -268,23 +328,26 @@ struct ImplicitDepLoader { struct DependencyScan { DependencyScan(State* state, BuildLog* build_log, DepsLog* deps_log, DiskInterface* disk_interface, - DepfileParserOptions const* depfile_parser_options) - : build_log_(build_log), - disk_interface_(disk_interface), - dep_loader_(state, deps_log, disk_interface, depfile_parser_options), - dyndep_loader_(state, disk_interface) {} - - /// Update the |dirty_| state of the given node by inspecting its input edge. + DepfileParserOptions const* depfile_parser_options, + Explanations* explanations) + : build_log_(build_log), disk_interface_(disk_interface), + dep_loader_(state, deps_log, disk_interface, depfile_parser_options, + explanations), + dyndep_loader_(state, disk_interface), explanations_(explanations) {} + + /// Update the |dirty_| state of the given nodes by transitively inspecting + /// their input edges. /// Examine inputs, outputs, and command lines to judge whether an edge /// needs to be re-run, and update outputs_ready_ and each outputs' |dirty_| /// state accordingly. + /// Appends any validation nodes found to the nodes parameter. /// Returns false on failure. - bool RecomputeDirty(Node* node, string* err); + bool RecomputeDirty(Node* node, std::vector* validation_nodes, std::string* err); /// Recompute whether any output of the edge is dirty, if so sets |*dirty|. /// Returns false on failure. bool RecomputeOutputsDirty(Edge* edge, Node* most_recent_input, - bool* dirty, string* err); + bool* dirty, std::string* err); BuildLog* build_log() const { return build_log_; @@ -301,22 +364,99 @@ struct DependencyScan { /// build graph with the new information. One overload accepts /// a caller-owned 'DyndepFile' object in which to store the /// information loaded from the dyndep file. - bool LoadDyndeps(Node* node, string* err) const; - bool LoadDyndeps(Node* node, DyndepFile* ddf, string* err) const; + bool LoadDyndeps(Node* node, std::string* err) const; + bool LoadDyndeps(Node* node, DyndepFile* ddf, std::string* err) const; private: - bool RecomputeDirty(Node* node, vector* stack, string* err); - bool VerifyDAG(Node* node, vector* stack, string* err); + bool RecomputeNodeDirty(Node* node, std::vector* stack, + std::vector* validation_nodes, std::string* err); + bool VerifyDAG(Node* node, std::vector* stack, std::string* err); /// Recompute whether a given single output should be marked dirty. /// Returns true if so. - bool RecomputeOutputDirty(Edge* edge, Node* most_recent_input, - const string& command, Node* output); + bool RecomputeOutputDirty(const Edge* edge, const Node* most_recent_input, + const std::string& command, Node* output); + + void RecordExplanation(const Node* node, const char* fmt, ...); BuildLog* build_log_; DiskInterface* disk_interface_; ImplicitDepLoader dep_loader_; DyndepLoader dyndep_loader_; + OptionalExplanations explanations_; +}; + +// Implements a less comparison for edges by priority, where highest +// priority is defined lexicographically first by largest critical +// time, then lowest ID. +// +// Including ID means that wherever the critical path weights are the +// same, the edges are executed in ascending ID order which was +// historically how all tasks were scheduled. +struct EdgePriorityLess { + bool operator()(const Edge* e1, const Edge* e2) const { + const int64_t cw1 = e1->critical_path_weight(); + const int64_t cw2 = e2->critical_path_weight(); + if (cw1 != cw2) { + return cw1 < cw2; + } + return e1->id_ > e2->id_; + } +}; + +// Reverse of EdgePriorityLess, e.g. to sort by highest priority first +struct EdgePriorityGreater { + bool operator()(const Edge* e1, const Edge* e2) const { + return EdgePriorityLess()(e2, e1); + } +}; + +// A priority queue holding non-owning Edge pointers. top() will +// return the edge with the largest critical path weight, and lowest +// ID if more than one edge has the same critical path weight. +class EdgePriorityQueue: + public std::priority_queue, EdgePriorityLess>{ +public: + void clear() { + c.clear(); + } +}; + +/// A class used to collect the transitive set of inputs from a given set +/// of starting nodes. Used to implement the `inputs` tool. +/// +/// When collecting inputs, the outputs of phony edges are always ignored +/// from the result, but are followed by the dependency walk. +/// +/// Usage is: +/// - Create instance. +/// - Call VisitNode() for each root node to collect inputs from. +/// - Call inputs() to retrieve the list of input node pointers. +/// - Call GetInputsAsStrings() to retrieve the list of inputs as a string +/// vector. +/// +struct InputsCollector { + /// Visit a single @arg node during this collection. + void VisitNode(const Node* node); + + /// Retrieve list of visited input nodes. A dependency always appears + /// before its dependents in the result, but final order depends on the + /// order of the VisitNode() calls performed before this. + const std::vector& inputs() const { return inputs_; } + + /// Same as inputs(), but returns the list of visited nodes as a list of + /// strings, with optional shell escaping. + std::vector GetInputsAsStrings(bool shell_escape = false) const; + + /// Reset collector state. + void Reset() { + inputs_.clear(); + visited_nodes_.clear(); + } + + private: + std::vector inputs_; + std::set visited_nodes_; }; #endif // NINJA_GRAPH_H_ diff --git a/src/graph_test.cc b/src/graph_test.cc index c8cca1c0df..d29118ae3a 100644 --- a/src/graph_test.cc +++ b/src/graph_test.cc @@ -13,12 +13,15 @@ // limitations under the License. #include "graph.h" -#include "build.h" +#include "build.h" +#include "command_collector.h" #include "test.h" +using namespace std; + struct GraphTest : public StateTestWithBuiltinRules { - GraphTest() : scan_(&state_, NULL, NULL, &fs_, NULL) {} + GraphTest() : scan_(&state_, NULL, NULL, &fs_, NULL, NULL) {} VirtualFileSystem fs_; DependencyScan scan_; @@ -31,7 +34,7 @@ TEST_F(GraphTest, MissingImplicit) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); // A missing implicit dep *should* make the output dirty. @@ -49,7 +52,7 @@ TEST_F(GraphTest, ModifiedImplicit) { fs_.Create("implicit", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); // A modified implicit dep should make the output dirty. @@ -69,7 +72,7 @@ TEST_F(GraphTest, FunkyMakefilePath) { fs_.Create("implicit.h", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); // implicit.h has changed, though our depfile refers to it with a @@ -92,7 +95,7 @@ TEST_F(GraphTest, ExplicitImplicit) { fs_.Create("data", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); // We have both an implicit and an explicit dep on implicit.h. @@ -106,7 +109,7 @@ TEST_F(GraphTest, ImplicitOutputParse) { "build out | out.imp: cat in\n")); Edge* edge = GetNode("out")->in_edge(); - EXPECT_EQ(2, edge->outputs_.size()); + EXPECT_EQ(size_t(2), edge->outputs_.size()); EXPECT_EQ("out", edge->outputs_[0]->path()); EXPECT_EQ("out.imp", edge->outputs_[1]->path()); EXPECT_EQ(1, edge->implicit_outs_); @@ -120,7 +123,7 @@ TEST_F(GraphTest, ImplicitOutputMissing) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out")->dirty()); @@ -136,7 +139,7 @@ TEST_F(GraphTest, ImplicitOutputOutOfDate) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out")->dirty()); @@ -148,7 +151,7 @@ TEST_F(GraphTest, ImplicitOutputOnlyParse) { "build | out.imp: cat in\n")); Edge* edge = GetNode("out.imp")->in_edge(); - EXPECT_EQ(1, edge->outputs_.size()); + EXPECT_EQ(size_t(1), edge->outputs_.size()); EXPECT_EQ("out.imp", edge->outputs_[0]->path()); EXPECT_EQ(1, edge->implicit_outs_); EXPECT_EQ(edge, GetNode("out.imp")->in_edge()); @@ -160,7 +163,7 @@ TEST_F(GraphTest, ImplicitOutputOnlyMissing) { fs_.Create("in", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out.imp")->dirty()); @@ -174,7 +177,7 @@ TEST_F(GraphTest, ImplicitOutputOnlyOutOfDate) { fs_.Create("in", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.imp"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out.imp")->dirty()); @@ -191,7 +194,7 @@ TEST_F(GraphTest, PathWithCurrentDirectory) { fs_.Create("out.o", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("out.o")->dirty()); @@ -213,12 +216,155 @@ TEST_F(GraphTest, RootNodes) { } } +TEST_F(GraphTest, InputsCollector) { + // Build plan for the following graph: + // + // in1 + // |___________ + // | | + // === === + // | | + // out1 mid1 + // | ____|_____ + // | | | + // | === ======= + // | | | | + // | out2 out3 out4 + // | | | + // =======phony====== + // | + // all + // + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out1: cat in1\n" + "build mid1: cat in1\n" + "build out2: cat mid1\n" + "build out3 out4: cat mid1\n" + "build all: phony out1 out2 out3\n")); + + InputsCollector collector; + + // Start visit from out1, this should add in1 to the inputs. + collector.Reset(); + collector.VisitNode(GetNode("out1")); + auto inputs = collector.GetInputsAsStrings(); + ASSERT_EQ(1u, inputs.size()); + EXPECT_EQ("in1", inputs[0]); + + // Add a visit from out2, this should add mid1. + collector.VisitNode(GetNode("out2")); + inputs = collector.GetInputsAsStrings(); + ASSERT_EQ(2u, inputs.size()); + EXPECT_EQ("in1", inputs[0]); + EXPECT_EQ("mid1", inputs[1]); + + // Another visit from all, this should add out1, out2 and out3, + // but not out4. + collector.VisitNode(GetNode("all")); + inputs = collector.GetInputsAsStrings(); + ASSERT_EQ(5u, inputs.size()); + EXPECT_EQ("in1", inputs[0]); + EXPECT_EQ("mid1", inputs[1]); + EXPECT_EQ("out1", inputs[2]); + EXPECT_EQ("out2", inputs[3]); + EXPECT_EQ("out3", inputs[4]); + + collector.Reset(); + + // Starting directly from all, will add out1 before mid1 compared + // to the previous example above. + collector.VisitNode(GetNode("all")); + inputs = collector.GetInputsAsStrings(); + ASSERT_EQ(5u, inputs.size()); + EXPECT_EQ("in1", inputs[0]); + EXPECT_EQ("out1", inputs[1]); + EXPECT_EQ("mid1", inputs[2]); + EXPECT_EQ("out2", inputs[3]); + EXPECT_EQ("out3", inputs[4]); +} + +TEST_F(GraphTest, InputsCollectorWithEscapes) { + ASSERT_NO_FATAL_FAILURE(AssertParse( + &state_, + "build out$ 1: cat in1 in2 in$ with$ space | implicit || order_only\n")); + + InputsCollector collector; + collector.VisitNode(GetNode("out 1")); + auto inputs = collector.GetInputsAsStrings(); + ASSERT_EQ(5u, inputs.size()); + EXPECT_EQ("in1", inputs[0]); + EXPECT_EQ("in2", inputs[1]); + EXPECT_EQ("in with space", inputs[2]); + EXPECT_EQ("implicit", inputs[3]); + EXPECT_EQ("order_only", inputs[4]); + + inputs = collector.GetInputsAsStrings(true); + ASSERT_EQ(5u, inputs.size()); + EXPECT_EQ("in1", inputs[0]); + EXPECT_EQ("in2", inputs[1]); +#ifdef _WIN32 + EXPECT_EQ("\"in with space\"", inputs[2]); +#else + EXPECT_EQ("'in with space'", inputs[2]); +#endif + EXPECT_EQ("implicit", inputs[3]); + EXPECT_EQ("order_only", inputs[4]); +} + +TEST_F(GraphTest, CommandCollector) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, + "build out1: cat in1\n" + "build mid1: cat in1\n" + "build out2: cat mid1\n" + "build out3 out4: cat mid1\n" + "build all: phony out1 out2 out3\n")); + { + CommandCollector collector; + auto& edges = collector.in_edges; + + // Start visit from out2; this should add `build mid1` and `build out2` to + // the edge list. + collector.CollectFrom(GetNode("out2")); + ASSERT_EQ(2u, edges.size()); + EXPECT_EQ("cat in1 > mid1", edges[0]->EvaluateCommand()); + EXPECT_EQ("cat mid1 > out2", edges[1]->EvaluateCommand()); + + // Add a visit from out1, this should append `build out1` + collector.CollectFrom(GetNode("out1")); + ASSERT_EQ(3u, edges.size()); + EXPECT_EQ("cat in1 > out1", edges[2]->EvaluateCommand()); + + // Another visit from all; this should add edges for out1, out2 and out3, + // but not all (because it's phony). + collector.CollectFrom(GetNode("all")); + ASSERT_EQ(4u, edges.size()); + EXPECT_EQ("cat in1 > mid1", edges[0]->EvaluateCommand()); + EXPECT_EQ("cat mid1 > out2", edges[1]->EvaluateCommand()); + EXPECT_EQ("cat in1 > out1", edges[2]->EvaluateCommand()); + EXPECT_EQ("cat mid1 > out3 out4", edges[3]->EvaluateCommand()); + } + + { + CommandCollector collector; + auto& edges = collector.in_edges; + + // Starting directly from all, will add `build out1` before `build mid1` + // compared to the previous example above. + collector.CollectFrom(GetNode("all")); + ASSERT_EQ(4u, edges.size()); + EXPECT_EQ("cat in1 > out1", edges[0]->EvaluateCommand()); + EXPECT_EQ("cat in1 > mid1", edges[1]->EvaluateCommand()); + EXPECT_EQ("cat mid1 > out2", edges[2]->EvaluateCommand()); + EXPECT_EQ("cat mid1 > out3 out4", edges[3]->EvaluateCommand()); + } +} + TEST_F(GraphTest, VarInOutPathEscaping) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build a$ b: cat no'space with$ space$$ no\"space2\n")); Edge* edge = GetNode("a b")->in_edge(); -#if _WIN32 +#ifdef _WIN32 EXPECT_EQ("cat no'space \"with space$\" \"no\\\"space2\" > \"a b\"", edge->EvaluateCommand()); #else @@ -239,7 +385,7 @@ TEST_F(GraphTest, DepfileWithCanonicalizablePath) { fs_.Create("out.o", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("out.o")->dirty()); @@ -259,13 +405,13 @@ TEST_F(GraphTest, DepfileRemoved) { fs_.Create("out.o", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("out.o")->dirty()); state_.Reset(); fs_.RemoveFile("out.o.d"); - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out.o"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("out.o")->dirty()); } @@ -312,7 +458,7 @@ TEST_F(GraphTest, NestedPhonyPrintsDone) { "build n2: phony n1\n" ); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("n2"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("n2"), NULL, &err)); ASSERT_EQ("", err); Plan plan_; @@ -331,7 +477,7 @@ TEST_F(GraphTest, PhonySelfReferenceError) { parser_opts); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> a [-w phonycycle=err]", err); } @@ -343,7 +489,7 @@ TEST_F(GraphTest, DependencyCycle) { "build pre: cat out\n"); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("dependency cycle: out -> mid -> in -> pre -> out", err); } @@ -351,7 +497,7 @@ TEST_F(GraphTest, CycleInEdgesButNotInNodes1) { string err; AssertParse(&state_, "build a b: cat a\n"); - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> a", err); } @@ -359,7 +505,7 @@ TEST_F(GraphTest, CycleInEdgesButNotInNodes2) { string err; ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build b a: cat a\n")); - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> a", err); } @@ -368,7 +514,7 @@ TEST_F(GraphTest, CycleInEdgesButNotInNodes3) { ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, "build a b: cat c\n" "build c: cat a\n")); - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("b"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> c -> a", err); } @@ -380,7 +526,7 @@ TEST_F(GraphTest, CycleInEdgesButNotInNodes4) { "build b: cat a\n" "build a e: cat d\n" "build f: cat e\n")); - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("f"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("f"), NULL, &err)); ASSERT_EQ("dependency cycle: a -> d -> c -> b -> a", err); } @@ -396,14 +542,14 @@ TEST_F(GraphTest, CycleWithLengthZeroFromDepfile) { fs_.Create("dep.d", "a: b\n"); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err)); ASSERT_EQ("dependency cycle: b -> b", err); // Despite the depfile causing edge to be a cycle (it has outputs a and b, // but the depfile also adds b as an input), the deps should have been loaded // only once: Edge* edge = GetNode("a")->in_edge(); - EXPECT_EQ(1, edge->inputs_.size()); + EXPECT_EQ(size_t(1), edge->inputs_.size()); EXPECT_EQ("b", edge->inputs_[0]->path()); } @@ -421,14 +567,14 @@ TEST_F(GraphTest, CycleWithLengthOneFromDepfile) { fs_.Create("dep.d", "a: c\n"); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("a"), NULL, &err)); ASSERT_EQ("dependency cycle: b -> c -> b", err); // Despite the depfile causing edge to be a cycle (|edge| has outputs a and b, // but c's in_edge has b as input but the depfile also adds |edge| as // output)), the deps should have been loaded only once: Edge* edge = GetNode("a")->in_edge(); - EXPECT_EQ(1, edge->inputs_.size()); + EXPECT_EQ(size_t(1), edge->inputs_.size()); EXPECT_EQ("c", edge->inputs_[0]->path()); } @@ -448,14 +594,14 @@ TEST_F(GraphTest, CycleWithLengthOneFromDepfileOneHopAway) { fs_.Create("dep.d", "a: c\n"); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("d"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("d"), NULL, &err)); ASSERT_EQ("dependency cycle: b -> c -> b", err); // Despite the depfile causing edge to be a cycle (|edge| has outputs a and b, // but c's in_edge has b as input but the depfile also adds |edge| as // output)), the deps should have been loaded only once: Edge* edge = GetNode("a")->in_edge(); - EXPECT_EQ(1, edge->inputs_.size()); + EXPECT_EQ(size_t(1), edge->inputs_.size()); EXPECT_EQ("c", edge->inputs_[0]->path()); } @@ -499,13 +645,44 @@ TEST_F(GraphTest, DyndepLoadTrivial) { EXPECT_FALSE(GetNode("dd")->dyndep_pending()); Edge* edge = GetNode("out")->in_edge(); - ASSERT_EQ(1u, edge->outputs_.size()); + ASSERT_EQ(size_t(1), edge->outputs_.size()); EXPECT_EQ("out", edge->outputs_[0]->path()); - ASSERT_EQ(2u, edge->inputs_.size()); + ASSERT_EQ(size_t(2), edge->inputs_.size()); EXPECT_EQ("in", edge->inputs_[0]->path()); EXPECT_EQ("dd", edge->inputs_[1]->path()); - EXPECT_EQ(0u, edge->implicit_deps_); - EXPECT_EQ(1u, edge->order_only_deps_); + EXPECT_EQ(0, edge->implicit_deps_); + EXPECT_EQ(1, edge->order_only_deps_); + EXPECT_FALSE(edge->GetBindingBool("restat")); +} + +TEST_F(GraphTest, DyndepLoadImplicit) { + AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out1: r in || dd\n" +" dyndep = dd\n" +"build out2: r in\n" + ); + fs_.Create("dd", +"ninja_dyndep_version = 1\n" +"build out1: dyndep | out2\n" + ); + + string err; + ASSERT_TRUE(GetNode("dd")->dyndep_pending()); + EXPECT_TRUE(scan_.LoadDyndeps(GetNode("dd"), &err)); + EXPECT_EQ("", err); + EXPECT_FALSE(GetNode("dd")->dyndep_pending()); + + Edge* edge = GetNode("out1")->in_edge(); + ASSERT_EQ(size_t(1), edge->outputs_.size()); + EXPECT_EQ("out1", edge->outputs_[0]->path()); + ASSERT_EQ(size_t(3), edge->inputs_.size()); + EXPECT_EQ("in", edge->inputs_[0]->path()); + EXPECT_EQ("out2", edge->inputs_[1]->path()); + EXPECT_EQ("dd", edge->inputs_[2]->path()); + EXPECT_EQ(1, edge->implicit_deps_); + EXPECT_EQ(1, edge->order_only_deps_); EXPECT_FALSE(edge->GetBindingBool("restat")); } @@ -631,35 +808,35 @@ TEST_F(GraphTest, DyndepLoadMultiple) { EXPECT_FALSE(GetNode("dd")->dyndep_pending()); Edge* edge1 = GetNode("out1")->in_edge(); - ASSERT_EQ(2u, edge1->outputs_.size()); + ASSERT_EQ(size_t(2), edge1->outputs_.size()); EXPECT_EQ("out1", edge1->outputs_[0]->path()); EXPECT_EQ("out1imp", edge1->outputs_[1]->path()); - EXPECT_EQ(1u, edge1->implicit_outs_); - ASSERT_EQ(3u, edge1->inputs_.size()); + EXPECT_EQ(1, edge1->implicit_outs_); + ASSERT_EQ(size_t(3), edge1->inputs_.size()); EXPECT_EQ("in1", edge1->inputs_[0]->path()); EXPECT_EQ("in1imp", edge1->inputs_[1]->path()); EXPECT_EQ("dd", edge1->inputs_[2]->path()); - EXPECT_EQ(1u, edge1->implicit_deps_); - EXPECT_EQ(1u, edge1->order_only_deps_); + EXPECT_EQ(1, edge1->implicit_deps_); + EXPECT_EQ(1, edge1->order_only_deps_); EXPECT_FALSE(edge1->GetBindingBool("restat")); EXPECT_EQ(edge1, GetNode("out1imp")->in_edge()); Node* in1imp = GetNode("in1imp"); - ASSERT_EQ(1u, in1imp->out_edges().size()); + ASSERT_EQ(size_t(1), in1imp->out_edges().size()); EXPECT_EQ(edge1, in1imp->out_edges()[0]); Edge* edge2 = GetNode("out2")->in_edge(); - ASSERT_EQ(1u, edge2->outputs_.size()); + ASSERT_EQ(size_t(1), edge2->outputs_.size()); EXPECT_EQ("out2", edge2->outputs_[0]->path()); - EXPECT_EQ(0u, edge2->implicit_outs_); - ASSERT_EQ(3u, edge2->inputs_.size()); + EXPECT_EQ(0, edge2->implicit_outs_); + ASSERT_EQ(size_t(3), edge2->inputs_.size()); EXPECT_EQ("in2", edge2->inputs_[0]->path()); EXPECT_EQ("in2imp", edge2->inputs_[1]->path()); EXPECT_EQ("dd", edge2->inputs_[2]->path()); - EXPECT_EQ(1u, edge2->implicit_deps_); - EXPECT_EQ(1u, edge2->order_only_deps_); + EXPECT_EQ(1, edge2->implicit_deps_); + EXPECT_EQ(1, edge2->order_only_deps_); EXPECT_TRUE(edge2->GetBindingBool("restat")); Node* in2imp = GetNode("in2imp"); - ASSERT_EQ(1u, in2imp->out_edges().size()); + ASSERT_EQ(size_t(1), in2imp->out_edges().size()); EXPECT_EQ(edge2, in2imp->out_edges()[0]); } @@ -672,7 +849,7 @@ TEST_F(GraphTest, DyndepFileMissing) { ); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("loading 'dd': No such file or directory", err); } @@ -688,7 +865,7 @@ TEST_F(GraphTest, DyndepFileError) { ); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("'out' not mentioned in its dyndep file 'dd'", err); } @@ -708,7 +885,7 @@ TEST_F(GraphTest, DyndepImplicitInputNewer) { fs_.Create("in", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("in")->dirty()); @@ -736,7 +913,7 @@ TEST_F(GraphTest, DyndepFileReady) { fs_.Create("in", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("in")->dirty()); @@ -761,7 +938,7 @@ TEST_F(GraphTest, DyndepFileNotClean) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("dd")->dirty()); @@ -787,7 +964,7 @@ TEST_F(GraphTest, DyndepFileNotReady) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_FALSE(GetNode("dd")->dirty()); @@ -815,7 +992,7 @@ TEST_F(GraphTest, DyndepFileSecondNotReady) { fs_.Create("out", ""); string err; - EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); ASSERT_EQ("", err); EXPECT_TRUE(GetNode("dd1")->dirty()); @@ -844,15 +1021,123 @@ TEST_F(GraphTest, DyndepFileCircular) { Edge* edge = GetNode("out")->in_edge(); string err; - EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), &err)); + EXPECT_FALSE(scan_.RecomputeDirty(GetNode("out"), NULL, &err)); EXPECT_EQ("dependency cycle: circ -> in -> circ", err); // Verify that "out.d" was loaded exactly once despite // circular reference discovered from dyndep file. - ASSERT_EQ(3u, edge->inputs_.size()); + ASSERT_EQ(size_t(3), edge->inputs_.size()); EXPECT_EQ("in", edge->inputs_[0]->path()); EXPECT_EQ("inimp", edge->inputs_[1]->path()); EXPECT_EQ("dd", edge->inputs_[2]->path()); - EXPECT_EQ(1u, edge->implicit_deps_); - EXPECT_EQ(1u, edge->order_only_deps_); + EXPECT_EQ(1, edge->implicit_deps_); + EXPECT_EQ(1, edge->order_only_deps_); +} + +TEST_F(GraphTest, Validation) { + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"build out: cat in |@ validate\n" +"build validate: cat in\n")); + + fs_.Create("in", ""); + string err; + std::vector validation_nodes; + EXPECT_TRUE(scan_.RecomputeDirty(GetNode("out"), &validation_nodes, &err)); + ASSERT_EQ("", err); + + ASSERT_EQ(validation_nodes.size(), size_t(1)); + EXPECT_EQ(validation_nodes[0]->path(), "validate"); + + EXPECT_TRUE(GetNode("out")->dirty()); + EXPECT_TRUE(GetNode("validate")->dirty()); +} + +// Check that phony's dependencies' mtimes are propagated. +TEST_F(GraphTest, PhonyDepsMtimes) { + string err; + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule touch\n" +" command = touch $out\n" +"build in_ph: phony in1\n" +"build out1: touch in_ph\n" +)); + fs_.Create("in1", ""); + fs_.Create("out1", ""); + Node* out1 = GetNode("out1"); + Node* in1 = GetNode("in1"); + + EXPECT_TRUE(scan_.RecomputeDirty(out1, NULL, &err)); + EXPECT_TRUE(!out1->dirty()); + + // Get the mtime of out1 + ASSERT_TRUE(in1->Stat(&fs_, &err)); + ASSERT_TRUE(out1->Stat(&fs_, &err)); + TimeStamp out1Mtime1 = out1->mtime(); + TimeStamp in1Mtime1 = in1->mtime(); + + // Touch in1. This should cause out1 to be dirty + state_.Reset(); + fs_.Tick(); + fs_.Create("in1", ""); + + ASSERT_TRUE(in1->Stat(&fs_, &err)); + EXPECT_GT(in1->mtime(), in1Mtime1); + + EXPECT_TRUE(scan_.RecomputeDirty(out1, NULL, &err)); + EXPECT_GT(in1->mtime(), in1Mtime1); + EXPECT_EQ(out1->mtime(), out1Mtime1); + EXPECT_TRUE(out1->dirty()); } + +// Test that EdgeQueue correctly prioritizes by critical time +TEST_F(GraphTest, EdgeQueuePriority) { + + ASSERT_NO_FATAL_FAILURE(AssertParse(&state_, +"rule r\n" +" command = unused\n" +"build out1: r in1\n" +"build out2: r in2\n" +"build out3: r in3\n" +)); + + const int n_edges = 3; + Edge *(edges)[n_edges] = { + GetNode("out1")->in_edge(), + GetNode("out2")->in_edge(), + GetNode("out3")->in_edge(), + }; + + // Output is largest critical time to smallest + for (int i = 0; i < n_edges; ++i) { + edges[i]->set_critical_path_weight(i * 10); + } + + EdgePriorityQueue queue; + for (int i = 0; i < n_edges; ++i) { + queue.push(edges[i]); + } + + EXPECT_EQ(queue.size(), static_cast(n_edges)); + for (int i = 0; i < n_edges; ++i) { + EXPECT_EQ(queue.top(), edges[n_edges - 1 - i]); + queue.pop(); + } + EXPECT_TRUE(queue.empty()); + + // When there is ambiguity, the lowest edge id comes first + for (int i = 0; i < n_edges; ++i) { + edges[i]->set_critical_path_weight(0); + } + + queue.push(edges[1]); + queue.push(edges[2]); + queue.push(edges[0]); + + for (int i = 0; i < n_edges; ++i) { + EXPECT_EQ(queue.top(), edges[i]); + queue.pop(); + } + EXPECT_TRUE(queue.empty()); +} + + diff --git a/src/graphviz.cc b/src/graphviz.cc index 0d07251251..37b7108962 100644 --- a/src/graphviz.cc +++ b/src/graphviz.cc @@ -20,6 +20,8 @@ #include "dyndep.h" #include "graph.h" +using namespace std; + void GraphViz::AddTarget(Node* node) { if (visited_nodes_.find(node) != visited_nodes_.end()) return; diff --git a/src/graphviz.h b/src/graphviz.h index 601c9b2ea0..3a3282e290 100644 --- a/src/graphviz.h +++ b/src/graphviz.h @@ -18,6 +18,7 @@ #include #include "dyndep.h" +#include "graph.h" struct DiskInterface; struct Node; @@ -34,7 +35,7 @@ struct GraphViz { DyndepLoader dyndep_loader_; std::set visited_nodes_; - std::set visited_edges_; + EdgeSet visited_edges_; }; #endif // NINJA_GRAPHVIZ_H_ diff --git a/src/hash_collision_bench.cc b/src/hash_collision_bench.cc index ff947dca60..8f37ed0d7b 100644 --- a/src/hash_collision_bench.cc +++ b/src/hash_collision_bench.cc @@ -15,20 +15,22 @@ #include "build_log.h" #include -using namespace std; #include #include +using namespace std; + int random(int low, int high) { return int(low + (rand() / double(RAND_MAX)) * (high - low) + 0.5); } void RandomCommand(char** s) { int len = random(5, 100); - *s = new char[len]; + *s = new char[len+1]; for (int i = 0; i < len; ++i) (*s)[i] = (char)random(32, 127); + (*s)[len] = '\0'; } int main() { diff --git a/src/hash_map.h b/src/hash_map.h index 55d2c9d46d..4361c80a35 100644 --- a/src/hash_map.h +++ b/src/hash_map.h @@ -20,41 +20,8 @@ #include "string_piece.h" #include "util.h" -// MurmurHash2, by Austin Appleby -static inline -unsigned int MurmurHash2(const void* key, size_t len) { - static const unsigned int seed = 0xDECAFBAD; - const unsigned int m = 0x5bd1e995; - const int r = 24; - unsigned int h = seed ^ len; - const unsigned char* data = (const unsigned char*)key; - while (len >= 4) { - unsigned int k; - memcpy(&k, data, sizeof k); - k *= m; - k ^= k >> r; - k *= m; - h *= m; - h ^= k; - data += 4; - len -= 4; - } - switch (len) { - case 3: h ^= data[2] << 16; - NINJA_FALLTHROUGH; - case 2: h ^= data[1] << 8; - NINJA_FALLTHROUGH; - case 1: h ^= data[0]; - h *= m; - }; - h ^= h >> 13; - h *= m; - h ^= h >> 15; - return h; -} - -#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) -#include +#include "third_party/emhash/hash_table8.hpp" +#include "third_party/rapidhash/rapidhash.h" namespace std { template<> @@ -63,47 +30,10 @@ struct hash { typedef size_t result_type; size_t operator()(StringPiece key) const { - return MurmurHash2(key.str_, key.len_); - } -}; -} - -#elif defined(_MSC_VER) -#include - -using stdext::hash_map; -using stdext::hash_compare; - -struct StringPieceCmp : public hash_compare { - size_t operator()(const StringPiece& key) const { - return MurmurHash2(key.str_, key.len_); - } - bool operator()(const StringPiece& a, const StringPiece& b) const { - int cmp = memcmp(a.str_, b.str_, min(a.len_, b.len_)); - if (cmp < 0) { - return true; - } else if (cmp > 0) { - return false; - } else { - return a.len_ < b.len_; - } - } -}; - -#else -#include - -using __gnu_cxx::hash_map; - -namespace __gnu_cxx { -template<> -struct hash { - size_t operator()(StringPiece key) const { - return MurmurHash2(key.str_, key.len_); + return rapidhash(key.str_, key.len_); } }; } -#endif /// A template for hash_maps keyed by a StringPiece whose string is /// owned externally (typically by the values). Use like: @@ -111,13 +41,7 @@ struct hash { /// mapping StringPiece => Foo*. template struct ExternalStringHashMap { -#if (__cplusplus >= 201103L) || (_MSC_VER >= 1900) - typedef std::unordered_map Type; -#elif defined(_MSC_VER) - typedef hash_map Type; -#else - typedef hash_map Type; -#endif + typedef emhash8::HashMap Type; }; #endif // NINJA_MAP_H_ diff --git a/src/includes_normalize-win32.cc b/src/includes_normalize-win32.cc index 79bf5b46a9..081e364ac3 100644 --- a/src/includes_normalize-win32.cc +++ b/src/includes_normalize-win32.cc @@ -24,6 +24,8 @@ #include +using namespace std; + namespace { bool InternalGetFullPathName(const StringPiece& file_name, char* buffer, @@ -46,7 +48,7 @@ bool IsPathSeparator(char c) { } // Return true if paths a and b are on the same windows drive. -// Return false if this funcation cannot check +// Return false if this function cannot check // whether or not on the same windows drive. bool SameDriveFast(StringPiece a, StringPiece b) { if (a.size() < 3 || b.size() < 3) { @@ -189,8 +191,7 @@ bool IncludesNormalize::Normalize(const string& input, } strncpy(copy, input.c_str(), input.size() + 1); uint64_t slash_bits; - if (!CanonicalizePath(copy, &len, &slash_bits, err)) - return false; + CanonicalizePath(copy, &len, &slash_bits); StringPiece partially_fixed(copy, len); string abs_input = AbsPath(partially_fixed, err); if (!err->empty()) diff --git a/src/includes_normalize.h b/src/includes_normalize.h index 0339581ec8..8d29a64a93 100644 --- a/src/includes_normalize.h +++ b/src/includes_normalize.h @@ -12,9 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#ifndef INCLUDES_NORMALIZE_H_ +#define INCLUDES_NORMALIZE_H_ + #include #include -using namespace std; struct StringPiece; @@ -22,18 +24,22 @@ struct StringPiece; /// TODO: this likely duplicates functionality of CanonicalizePath; refactor. struct IncludesNormalize { /// Normalize path relative to |relative_to|. - IncludesNormalize(const string& relative_to); + IncludesNormalize(const std::string& relative_to); // Internal utilities made available for testing, maybe useful otherwise. - static string AbsPath(StringPiece s, string* err); - static string Relativize(StringPiece path, - const vector& start_list, string* err); + static std::string AbsPath(StringPiece s, std::string* err); + static std::string Relativize(StringPiece path, + const std::vector& start_list, + std::string* err); /// Normalize by fixing slashes style, fixing redundant .. and . and makes the /// path |input| relative to |this->relative_to_| and store to |result|. - bool Normalize(const string& input, string* result, string* err) const; + bool Normalize(const std::string& input, std::string* result, + std::string* err) const; private: - string relative_to_; - vector split_relative_to_; + std::string relative_to_; + std::vector split_relative_to_; }; + +#endif // INCLUDES_NORMALIZE_H_ diff --git a/src/includes_normalize_test.cc b/src/includes_normalize_test.cc index dbcdbe0eb8..659d17032c 100644 --- a/src/includes_normalize_test.cc +++ b/src/includes_normalize_test.cc @@ -22,6 +22,8 @@ #include "test.h" #include "util.h" +using namespace std; + namespace { string GetCurDir() { @@ -115,10 +117,10 @@ TEST(IncludesNormalize, LongInvalidPath) { // Construct max size path having cwd prefix. // kExactlyMaxPath = "$cwd\\a\\aaaa...aaaa\0"; char kExactlyMaxPath[_MAX_PATH + 1]; - ASSERT_NE(_getcwd(kExactlyMaxPath, sizeof kExactlyMaxPath), NULL); + ASSERT_STRNE(_getcwd(kExactlyMaxPath, sizeof kExactlyMaxPath), NULL); int cwd_len = strlen(kExactlyMaxPath); - ASSERT_LE(cwd_len + 3 + 1, _MAX_PATH) + ASSERT_LE(cwd_len + 3 + 1, _MAX_PATH); kExactlyMaxPath[cwd_len] = '\\'; kExactlyMaxPath[cwd_len + 1] = 'a'; kExactlyMaxPath[cwd_len + 2] = '\\'; @@ -133,7 +135,8 @@ TEST(IncludesNormalize, LongInvalidPath) { } kExactlyMaxPath[_MAX_PATH] = '\0'; - EXPECT_EQ(strlen(kExactlyMaxPath), _MAX_PATH); + // This is a relatively safe cast as we can expect that _MAX_PATH will never be negative + EXPECT_EQ(strlen(kExactlyMaxPath), static_cast(_MAX_PATH)); string forward_slashes(kExactlyMaxPath); replace(forward_slashes.begin(), forward_slashes.end(), '\\', '/'); @@ -159,7 +162,7 @@ TEST(IncludesNormalize, ShortRelativeButTooLongAbsolutePath) { kExactlyMaxPath[i] = 'a'; } kExactlyMaxPath[_MAX_PATH] = '\0'; - EXPECT_EQ(strlen(kExactlyMaxPath), _MAX_PATH); + EXPECT_EQ(strlen(kExactlyMaxPath), static_cast(_MAX_PATH)); // Make sure a path that's exactly _MAX_PATH long fails with a proper error. EXPECT_FALSE(normalizer.Normalize(kExactlyMaxPath, &result, &err)); diff --git a/src/inline.sh b/src/inline.sh index b64e8cadf0..5092fa2b17 100755 --- a/src/inline.sh +++ b/src/inline.sh @@ -19,7 +19,14 @@ # stdin and writes stdout. varname="$1" -echo "const char $varname[] =" -od -t x1 -A n -v | sed -e 's|^[\t ]\{0,\}$||g; s|[\t ]\{1,\}| |g; s| \{1,\}$||g; s| |\\x|g; s|^|"|; s|$|"|' -echo ";" +# 'od' and 'sed' may not be available on all platforms, and may not support the +# flags used here. We must ensure that the script exits with a non-zero exit +# code in those cases. +byte_vals=$(od -t x1 -A n -v) || exit 1 +escaped_byte_vals=$(echo "${byte_vals}" \ + | sed -e 's|^[\t ]\{0,\}$||g; s|[\t ]\{1,\}| |g; s| \{1,\}$||g; s| |\\x|g; s|^|"|; s|$|"|') \ + || exit 1 + +# Only write output once we have successfully generated the required data +printf "const char %s[] = \n%s;" "${varname}" "${escaped_byte_vals}" diff --git a/src/json.cc b/src/json.cc new file mode 100644 index 0000000000..4bbf6e15ad --- /dev/null +++ b/src/json.cc @@ -0,0 +1,53 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "json.h" + +#include +#include + +std::string EncodeJSONString(const std::string& in) { + static const char* hex_digits = "0123456789abcdef"; + std::string out; + out.reserve(in.length() * 1.2); + for (std::string::const_iterator it = in.begin(); it != in.end(); ++it) { + char c = *it; + if (c == '\b') + out += "\\b"; + else if (c == '\f') + out += "\\f"; + else if (c == '\n') + out += "\\n"; + else if (c == '\r') + out += "\\r"; + else if (c == '\t') + out += "\\t"; + else if (0x0 <= c && c < 0x20) { + out += "\\u00"; + out += hex_digits[c >> 4]; + out += hex_digits[c & 0xf]; + } else if (c == '\\') + out += "\\\\"; + else if (c == '\"') + out += "\\\""; + else + out += c; + } + return out; +} + +void PrintJSONString(const std::string& in) { + std::string out = EncodeJSONString(in); + fwrite(out.c_str(), 1, out.length(), stdout); +} diff --git a/src/json.h b/src/json.h new file mode 100644 index 0000000000..3e5cf7817c --- /dev/null +++ b/src/json.h @@ -0,0 +1,26 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_JSON_H_ +#define NINJA_JSON_H_ + +#include + +// Encode a string in JSON format without enclosing quotes +std::string EncodeJSONString(const std::string& in); + +// Print a string in JSON format to stdout without enclosing quotes +void PrintJSONString(const std::string& in); + +#endif diff --git a/src/json_test.cc b/src/json_test.cc new file mode 100644 index 0000000000..b4afc73695 --- /dev/null +++ b/src/json_test.cc @@ -0,0 +1,40 @@ +// Copyright 2021 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "json.h" + +#include "test.h" + +TEST(JSONTest, RegularAscii) { + EXPECT_EQ(EncodeJSONString("foo bar"), "foo bar"); +} + +TEST(JSONTest, EscapedChars) { + EXPECT_EQ(EncodeJSONString("\"\\\b\f\n\r\t"), + "\\\"" + "\\\\" + "\\b\\f\\n\\r\\t"); +} + +// codepoints between 0 and 0x1f should be escaped +TEST(JSONTest, ControlChars) { + EXPECT_EQ(EncodeJSONString("\x01\x1f"), "\\u0001\\u001f"); +} + +// Leave them alone as JSON accepts unicode literals +// out of control character range +TEST(JSONTest, UTF8) { + const char* utf8str = "\xe4\xbd\xa0\xe5\xa5\xbd"; + EXPECT_EQ(EncodeJSONString(utf8str), utf8str); +} diff --git a/src/lexer.cc b/src/lexer.cc index 35ae97bc58..e5729f00a0 100644 --- a/src/lexer.cc +++ b/src/lexer.cc @@ -1,4 +1,4 @@ -/* Generated by re2c 0.16 */ +/* Generated by re2c */ // Copyright 2011 Google Inc. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -20,6 +20,8 @@ #include "eval_env.h" #include "util.h" +using namespace std; + bool Lexer::Error(const string& message, string* err) { // Compute line/column. int line = 1; @@ -83,6 +85,7 @@ const char* Lexer::TokenName(Token t) { case NEWLINE: return "newline"; case PIPE2: return "'||'"; case PIPE: return "'|'"; + case PIPEAT: return "'|@'"; case POOL: return "'pool'"; case RULE: return "'rule'"; case SUBNINJA: return "'subninja'"; @@ -233,8 +236,7 @@ Lexer::Token Lexer::ReadToken() { goto yy5; yy9: yyaccept = 0; - q = ++p; - yych = *p; + yych = *(q = ++p); if (yybm[0+yych] & 32) { goto yy9; } @@ -252,8 +254,7 @@ Lexer::Token Lexer::ReadToken() { if (yych <= 0x00) goto yy5; goto yy33; yy13: - ++p; - yych = *p; + yych = *++p; yy14: if (yybm[0+yych] & 64) { goto yy13; @@ -290,8 +291,9 @@ Lexer::Token Lexer::ReadToken() { if (yych == 'u') goto yy41; goto yy14; yy26: - ++p; - if ((yych = *p) == '|') goto yy42; + yych = *++p; + if (yych == '@') goto yy42; + if (yych == '|') goto yy44; { token = PIPE; break; } yy28: ++p; @@ -307,8 +309,7 @@ Lexer::Token Lexer::ReadToken() { goto yy5; } yy32: - ++p; - yych = *p; + yych = *++p; yy33: if (yybm[0+yych] & 128) { goto yy32; @@ -318,130 +319,133 @@ Lexer::Token Lexer::ReadToken() { { continue; } yy36: yych = *++p; - if (yych == 'i') goto yy44; + if (yych == 'i') goto yy46; goto yy14; yy37: yych = *++p; - if (yych == 'f') goto yy45; + if (yych == 'f') goto yy47; goto yy14; yy38: yych = *++p; - if (yych == 'c') goto yy46; + if (yych == 'c') goto yy48; goto yy14; yy39: yych = *++p; - if (yych == 'o') goto yy47; + if (yych == 'o') goto yy49; goto yy14; yy40: yych = *++p; - if (yych == 'l') goto yy48; + if (yych == 'l') goto yy50; goto yy14; yy41: yych = *++p; - if (yych == 'b') goto yy49; + if (yych == 'b') goto yy51; goto yy14; yy42: ++p; - { token = PIPE2; break; } + { token = PIPEAT; break; } yy44: - yych = *++p; - if (yych == 'l') goto yy50; - goto yy14; -yy45: - yych = *++p; - if (yych == 'a') goto yy51; - goto yy14; + ++p; + { token = PIPE2; break; } yy46: yych = *++p; if (yych == 'l') goto yy52; goto yy14; yy47: yych = *++p; - if (yych == 'l') goto yy53; + if (yych == 'a') goto yy53; goto yy14; yy48: yych = *++p; - if (yych == 'e') goto yy55; + if (yych == 'l') goto yy54; goto yy14; yy49: yych = *++p; - if (yych == 'n') goto yy57; + if (yych == 'l') goto yy55; goto yy14; yy50: yych = *++p; - if (yych == 'd') goto yy58; + if (yych == 'e') goto yy57; goto yy14; yy51: yych = *++p; - if (yych == 'u') goto yy60; + if (yych == 'n') goto yy59; goto yy14; yy52: yych = *++p; - if (yych == 'u') goto yy61; + if (yych == 'd') goto yy60; goto yy14; yy53: - ++p; - if (yybm[0+(yych = *p)] & 64) { + yych = *++p; + if (yych == 'u') goto yy62; + goto yy14; +yy54: + yych = *++p; + if (yych == 'u') goto yy63; + goto yy14; +yy55: + yych = *++p; + if (yybm[0+yych] & 64) { goto yy13; } { token = POOL; break; } -yy55: - ++p; - if (yybm[0+(yych = *p)] & 64) { +yy57: + yych = *++p; + if (yybm[0+yych] & 64) { goto yy13; } { token = RULE; break; } -yy57: +yy59: yych = *++p; - if (yych == 'i') goto yy62; + if (yych == 'i') goto yy64; goto yy14; -yy58: - ++p; - if (yybm[0+(yych = *p)] & 64) { +yy60: + yych = *++p; + if (yybm[0+yych] & 64) { goto yy13; } { token = BUILD; break; } -yy60: - yych = *++p; - if (yych == 'l') goto yy63; - goto yy14; -yy61: - yych = *++p; - if (yych == 'd') goto yy64; - goto yy14; yy62: yych = *++p; - if (yych == 'n') goto yy65; + if (yych == 'l') goto yy65; goto yy14; yy63: yych = *++p; - if (yych == 't') goto yy66; + if (yych == 'd') goto yy66; goto yy14; yy64: yych = *++p; - if (yych == 'e') goto yy68; + if (yych == 'n') goto yy67; goto yy14; yy65: yych = *++p; - if (yych == 'j') goto yy70; + if (yych == 't') goto yy68; goto yy14; yy66: - ++p; - if (yybm[0+(yych = *p)] & 64) { + yych = *++p; + if (yych == 'e') goto yy70; + goto yy14; +yy67: + yych = *++p; + if (yych == 'j') goto yy72; + goto yy14; +yy68: + yych = *++p; + if (yybm[0+yych] & 64) { goto yy13; } { token = DEFAULT; break; } -yy68: - ++p; - if (yybm[0+(yych = *p)] & 64) { +yy70: + yych = *++p; + if (yybm[0+yych] & 64) { goto yy13; } { token = INCLUDE; break; } -yy70: +yy72: yych = *++p; if (yych != 'a') goto yy14; - ++p; - if (yybm[0+(yych = *p)] & 64) { + yych = *++p; + if (yybm[0+yych] & 64) { goto yy13; } { token = SUBNINJA; break; } @@ -508,39 +512,38 @@ void Lexer::EatWhitespace() { }; yych = *p; if (yybm[0+yych] & 128) { - goto yy79; + goto yy81; } - if (yych <= 0x00) goto yy75; - if (yych == '$') goto yy82; - goto yy77; -yy75: - ++p; - { break; } + if (yych <= 0x00) goto yy77; + if (yych == '$') goto yy84; + goto yy79; yy77: ++p; -yy78: { break; } yy79: ++p; - yych = *p; +yy80: + { break; } +yy81: + yych = *++p; if (yybm[0+yych] & 128) { - goto yy79; + goto yy81; } { continue; } -yy82: +yy84: yych = *(q = ++p); - if (yych == '\n') goto yy83; - if (yych == '\r') goto yy85; - goto yy78; -yy83: + if (yych == '\n') goto yy85; + if (yych == '\r') goto yy87; + goto yy80; +yy85: ++p; { continue; } -yy85: +yy87: yych = *++p; - if (yych == '\n') goto yy87; + if (yych == '\n') goto yy89; p = q; - goto yy78; -yy87: + goto yy80; +yy89: ++p; { continue; } } @@ -592,18 +595,17 @@ bool Lexer::ReadIdent(string* out) { }; yych = *p; if (yybm[0+yych] & 128) { - goto yy93; + goto yy95; } ++p; { last_token_ = start; return false; } -yy93: - ++p; - yych = *p; +yy95: + yych = *++p; if (yybm[0+yych] & 128) { - goto yy93; + goto yy95; } { out->assign(start, p - start); @@ -663,34 +665,33 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { }; yych = *p; if (yybm[0+yych] & 16) { - goto yy100; + goto yy102; } if (yych <= '\r') { - if (yych <= 0x00) goto yy98; - if (yych <= '\n') goto yy103; - goto yy105; + if (yych <= 0x00) goto yy100; + if (yych <= '\n') goto yy105; + goto yy107; } else { - if (yych <= ' ') goto yy103; - if (yych <= '$') goto yy107; - goto yy103; + if (yych <= ' ') goto yy105; + if (yych <= '$') goto yy109; + goto yy105; } -yy98: +yy100: ++p; { last_token_ = start; return Error("unexpected EOF", err); } -yy100: - ++p; - yych = *p; +yy102: + yych = *++p; if (yybm[0+yych] & 16) { - goto yy100; + goto yy102; } { eval->AddText(StringPiece(start, p - start)); continue; } -yy103: +yy105: ++p; { if (path) { @@ -703,116 +704,112 @@ bool Lexer::ReadEvalString(EvalString* eval, bool path, string* err) { continue; } } -yy105: - ++p; - if ((yych = *p) == '\n') goto yy108; +yy107: + yych = *++p; + if (yych == '\n') goto yy110; { last_token_ = start; return Error(DescribeLastError(), err); } -yy107: +yy109: yych = *++p; if (yybm[0+yych] & 64) { - goto yy120; + goto yy122; } if (yych <= ' ') { if (yych <= '\f') { - if (yych == '\n') goto yy112; - goto yy110; + if (yych == '\n') goto yy114; + goto yy112; } else { - if (yych <= '\r') goto yy115; - if (yych <= 0x1F) goto yy110; - goto yy116; + if (yych <= '\r') goto yy117; + if (yych <= 0x1F) goto yy112; + goto yy118; } } else { if (yych <= '/') { - if (yych == '$') goto yy118; - goto yy110; + if (yych == '$') goto yy120; + goto yy112; } else { - if (yych <= ':') goto yy123; - if (yych <= '`') goto yy110; - if (yych <= '{') goto yy125; - goto yy110; + if (yych <= ':') goto yy125; + if (yych <= '`') goto yy112; + if (yych <= '{') goto yy127; + goto yy112; } } -yy108: +yy110: ++p; { if (path) p = start; break; } -yy110: +yy112: ++p; -yy111: +yy113: { last_token_ = start; return Error("bad $-escape (literal $ must be written as $$)", err); } -yy112: - ++p; - yych = *p; +yy114: + yych = *++p; if (yybm[0+yych] & 32) { - goto yy112; + goto yy114; } { continue; } -yy115: +yy117: yych = *++p; - if (yych == '\n') goto yy126; - goto yy111; -yy116: + if (yych == '\n') goto yy128; + goto yy113; +yy118: ++p; { eval->AddText(StringPiece(" ", 1)); continue; } -yy118: +yy120: ++p; { eval->AddText(StringPiece("$", 1)); continue; } -yy120: - ++p; - yych = *p; +yy122: + yych = *++p; if (yybm[0+yych] & 64) { - goto yy120; + goto yy122; } { eval->AddSpecial(StringPiece(start + 1, p - start - 1)); continue; } -yy123: +yy125: ++p; { eval->AddText(StringPiece(":", 1)); continue; } -yy125: +yy127: yych = *(q = ++p); if (yybm[0+yych] & 128) { - goto yy129; + goto yy131; } - goto yy111; -yy126: - ++p; - yych = *p; - if (yych == ' ') goto yy126; + goto yy113; +yy128: + yych = *++p; + if (yych == ' ') goto yy128; { continue; } -yy129: - ++p; - yych = *p; +yy131: + yych = *++p; if (yybm[0+yych] & 128) { - goto yy129; + goto yy131; } - if (yych == '}') goto yy132; + if (yych == '}') goto yy134; p = q; - goto yy111; -yy132: + goto yy113; +yy134: ++p; { eval->AddSpecial(StringPiece(start + 2, p - start - 3)); diff --git a/src/lexer.h b/src/lexer.h index f366556afc..683fd6c6ab 100644 --- a/src/lexer.h +++ b/src/lexer.h @@ -41,6 +41,7 @@ struct Lexer { NEWLINE, PIPE, PIPE2, + PIPEAT, POOL, RULE, SUBNINJA, @@ -55,7 +56,7 @@ struct Lexer { /// If the last token read was an ERROR token, provide more info /// or the empty string. - string DescribeLastError(); + std::string DescribeLastError(); /// Start parsing some input. void Start(StringPiece filename, StringPiece input); @@ -71,30 +72,30 @@ struct Lexer { /// Read a simple identifier (a rule or variable name). /// Returns false if a name can't be read. - bool ReadIdent(string* out); + bool ReadIdent(std::string* out); /// Read a path (complete with $escapes). /// Returns false only on error, returned path may be empty if a delimiter /// (space, newline) is hit. - bool ReadPath(EvalString* path, string* err) { + bool ReadPath(EvalString* path, std::string* err) { return ReadEvalString(path, true, err); } /// Read the value side of a var = value line (complete with $escapes). /// Returns false only on error. - bool ReadVarValue(EvalString* value, string* err) { + bool ReadVarValue(EvalString* value, std::string* err) { return ReadEvalString(value, false, err); } /// Construct an error message with context. - bool Error(const string& message, string* err); + bool Error(const std::string& message, std::string* err); private: /// Skip past whitespace (called after each read token/ident/etc.). void EatWhitespace(); /// Read a $-escaped string. - bool ReadEvalString(EvalString* eval, bool path, string* err); + bool ReadEvalString(EvalString* eval, bool path, std::string* err); StringPiece filename_; StringPiece input_; diff --git a/src/lexer.in.cc b/src/lexer.in.cc index c1fb8227cc..6f1d8e7937 100644 --- a/src/lexer.in.cc +++ b/src/lexer.in.cc @@ -19,6 +19,8 @@ #include "eval_env.h" #include "util.h" +using namespace std; + bool Lexer::Error(const string& message, string* err) { // Compute line/column. int line = 1; @@ -82,6 +84,7 @@ const char* Lexer::TokenName(Token t) { case NEWLINE: return "newline"; case PIPE2: return "'||'"; case PIPE: return "'|'"; + case PIPEAT: return "'|@'"; case POOL: return "'pool'"; case RULE: return "'rule'"; case SUBNINJA: return "'subninja'"; @@ -140,6 +143,7 @@ Lexer::Token Lexer::ReadToken() { "default" { token = DEFAULT; break; } "=" { token = EQUALS; break; } ":" { token = COLON; break; } + "|@" { token = PIPEAT; break; } "||" { token = PIPE2; break; } "|" { token = PIPE; break; } "include" { token = INCLUDE; break; } diff --git a/src/lexer_test.cc b/src/lexer_test.cc index 331d8e1ea9..c5c416dc54 100644 --- a/src/lexer_test.cc +++ b/src/lexer_test.cc @@ -17,6 +17,8 @@ #include "eval_env.h" #include "test.h" +using namespace std; + TEST(Lexer, ReadVarValue) { Lexer lexer("plain text $var $VaR ${x}\n"); EvalString eval; diff --git a/src/line_printer.cc b/src/line_printer.cc index 55469d9837..4a7b0bbf70 100644 --- a/src/line_printer.cc +++ b/src/line_printer.cc @@ -28,40 +28,40 @@ #include #endif +#include "elide_middle.h" #include "util.h" +using namespace std; + LinePrinter::LinePrinter() : have_blank_line_(true), console_locked_(false) { const char* term = getenv("TERM"); #ifndef _WIN32 smart_terminal_ = isatty(1) && term && string(term) != "dumb"; #else - // Disable output buffer. It'd be nice to use line buffering but - // MSDN says: "For some systems, [_IOLBF] provides line - // buffering. However, for Win32, the behavior is the same as _IOFBF - // - Full Buffering." if (term && string(term) == "dumb") { smart_terminal_ = false; } else { - setvbuf(stdout, NULL, _IONBF, 0); console_ = GetStdHandle(STD_OUTPUT_HANDLE); CONSOLE_SCREEN_BUFFER_INFO csbi; smart_terminal_ = GetConsoleScreenBufferInfo(console_, &csbi); } #endif supports_color_ = smart_terminal_; - if (!supports_color_) { - const char* clicolor_force = getenv("CLICOLOR_FORCE"); - supports_color_ = clicolor_force && string(clicolor_force) != "0"; - } #ifdef _WIN32 // Try enabling ANSI escape sequence support on Windows 10 terminals. if (supports_color_) { DWORD mode; if (GetConsoleMode(console_, &mode)) { - SetConsoleMode(console_, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING); + if (!SetConsoleMode(console_, mode | ENABLE_VIRTUAL_TERMINAL_PROCESSING)) { + supports_color_ = false; + } } } #endif + if (!supports_color_) { + const char* clicolor_force = getenv("CLICOLOR_FORCE"); + supports_color_ = clicolor_force && std::string(clicolor_force) != "0"; + } } void LinePrinter::Print(string to_print, LineType type) { @@ -82,29 +82,34 @@ void LinePrinter::Print(string to_print, LineType type) { CONSOLE_SCREEN_BUFFER_INFO csbi; GetConsoleScreenBufferInfo(console_, &csbi); - to_print = ElideMiddle(to_print, static_cast(csbi.dwSize.X)); - // We don't want to have the cursor spamming back and forth, so instead of - // printf use WriteConsoleOutput which updates the contents of the buffer, - // but doesn't move the cursor position. - COORD buf_size = { csbi.dwSize.X, 1 }; - COORD zero_zero = { 0, 0 }; - SMALL_RECT target = { - csbi.dwCursorPosition.X, csbi.dwCursorPosition.Y, - static_cast(csbi.dwCursorPosition.X + csbi.dwSize.X - 1), - csbi.dwCursorPosition.Y - }; - vector char_data(csbi.dwSize.X); - for (size_t i = 0; i < static_cast(csbi.dwSize.X); ++i) { - char_data[i].Char.AsciiChar = i < to_print.size() ? to_print[i] : ' '; - char_data[i].Attributes = csbi.wAttributes; + ElideMiddleInPlace(to_print, static_cast(csbi.dwSize.X)); + if (supports_color_) { // this means ENABLE_VIRTUAL_TERMINAL_PROCESSING + // succeeded + printf("%s\x1B[K", to_print.c_str()); // Clear to end of line. + fflush(stdout); + } else { + // We don't want to have the cursor spamming back and forth, so instead of + // printf use WriteConsoleOutput which updates the contents of the buffer, + // but doesn't move the cursor position. + COORD buf_size = { csbi.dwSize.X, 1 }; + COORD zero_zero = { 0, 0 }; + SMALL_RECT target = { csbi.dwCursorPosition.X, csbi.dwCursorPosition.Y, + static_cast(csbi.dwCursorPosition.X + + csbi.dwSize.X - 1), + csbi.dwCursorPosition.Y }; + vector char_data(csbi.dwSize.X); + for (size_t i = 0; i < static_cast(csbi.dwSize.X); ++i) { + char_data[i].Char.AsciiChar = i < to_print.size() ? to_print[i] : ' '; + char_data[i].Attributes = csbi.wAttributes; + } + WriteConsoleOutput(console_, &char_data[0], buf_size, zero_zero, &target); } - WriteConsoleOutput(console_, &char_data[0], buf_size, zero_zero, &target); #else // Limit output to width of the terminal if provided so we don't cause // line-wrapping. winsize size; if ((ioctl(STDOUT_FILENO, TIOCGWINSZ, &size) == 0) && size.ws_col) { - to_print = ElideMiddle(to_print, size.ws_col); + ElideMiddleInPlace(to_print, size.ws_col); } printf("%s", to_print.c_str()); printf("\x1B[K"); // Clear to end of line. @@ -114,6 +119,7 @@ void LinePrinter::Print(string to_print, LineType type) { have_blank_line_ = false; } else { printf("%s\n", to_print.c_str()); + fflush(stdout); } } diff --git a/src/line_printer.h b/src/line_printer.h index 92d4dc4480..a8ec9ff40e 100644 --- a/src/line_printer.h +++ b/src/line_printer.h @@ -17,7 +17,6 @@ #include #include -using namespace std; /// Prints lines of text, possibly overprinting previously printed lines /// if the terminal supports it. @@ -35,10 +34,10 @@ struct LinePrinter { }; /// Overprints the current line. If type is ELIDE, elides to_print to fit on /// one line. - void Print(string to_print, LineType type); + void Print(std::string to_print, LineType type); /// Prints a string on a new line, not overprinting previous output. - void PrintOnNewLine(const string& to_print); + void PrintOnNewLine(const std::string& to_print); /// Lock or unlock the console. Any output sent to the LinePrinter while the /// console is locked will not be printed until it is unlocked. @@ -58,13 +57,13 @@ struct LinePrinter { bool console_locked_; /// Buffered current line while console is locked. - string line_buffer_; + std::string line_buffer_; /// Buffered line type while console is locked. LineType line_type_; /// Buffered console output while console is locked. - string output_buffer_; + std::string output_buffer_; #ifdef _WIN32 void* console_; diff --git a/src/load_status.h b/src/load_status.h new file mode 100644 index 0000000000..0b16b1add3 --- /dev/null +++ b/src/load_status.h @@ -0,0 +1,24 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_LOAD_STATUS_H_ +#define NINJA_LOAD_STATUS_H_ + +enum LoadStatus { + LOAD_ERROR, + LOAD_SUCCESS, + LOAD_NOT_FOUND, +}; + +#endif // NINJA_LOAD_STATUS_H_ diff --git a/src/manifest_parser.cc b/src/manifest_parser.cc index 2011368301..04b31918f4 100644 --- a/src/manifest_parser.cc +++ b/src/manifest_parser.cc @@ -14,8 +14,11 @@ #include "manifest_parser.h" +#include #include #include + +#include #include #include "graph.h" @@ -23,6 +26,8 @@ #include "util.h" #include "version.h" +using namespace std; + ManifestParser::ManifestParser(State* state, FileReader* file_reader, ManifestParserOptions options) : Parser(state, file_reader), @@ -139,7 +144,7 @@ bool ManifestParser::ParseRule(string* err) { if (env_->LookupRuleCurrentScope(name) != NULL) return lexer_.Error("duplicate rule '" + name + "'", err); - Rule* rule = new Rule(name); // XXX scoped_ptr + auto rule = std::unique_ptr(new Rule(name)); while (lexer_.PeekToken(Lexer::INDENT)) { string key; @@ -165,7 +170,7 @@ bool ManifestParser::ParseRule(string* err) { if (rule->bindings_["command"].empty()) return lexer_.Error("expected 'command =' line", err); - env_->AddRule(rule); + env_->AddRule(std::move(rule)); return true; } @@ -188,33 +193,33 @@ bool ManifestParser::ParseDefault(string* err) { do { string path = eval.Evaluate(env_); - string path_err; + if (path.empty()) + return lexer_.Error("empty path", err); uint64_t slash_bits; // Unused because this only does lookup. - if (!CanonicalizePath(&path, &slash_bits, &path_err)) - return lexer_.Error(path_err, err); - if (!state_->AddDefault(path, &path_err)) - return lexer_.Error(path_err, err); + CanonicalizePath(&path, &slash_bits); + std::string default_err; + if (!state_->AddDefault(path, &default_err)) + return lexer_.Error(default_err, err); eval.Clear(); if (!lexer_.ReadPath(&eval, err)) return false; } while (!eval.empty()); - if (!ExpectToken(Lexer::NEWLINE, err)) - return false; - - return true; + return ExpectToken(Lexer::NEWLINE, err); } bool ManifestParser::ParseEdge(string* err) { - vector ins, outs; + ins_.clear(); + outs_.clear(); + validations_.clear(); { EvalString out; if (!lexer_.ReadPath(&out, err)) return false; while (!out.empty()) { - outs.push_back(out); + outs_.push_back(std::move(out)); out.Clear(); if (!lexer_.ReadPath(&out, err)) @@ -228,15 +233,15 @@ bool ManifestParser::ParseEdge(string* err) { for (;;) { EvalString out; if (!lexer_.ReadPath(&out, err)) - return err; + return false; if (out.empty()) break; - outs.push_back(out); + outs_.push_back(std::move(out)); ++implicit_outs; } } - if (outs.empty()) + if (outs_.empty()) return lexer_.Error("expected path", err); if (!ExpectToken(Lexer::COLON, err)) @@ -257,7 +262,7 @@ bool ManifestParser::ParseEdge(string* err) { return false; if (in.empty()) break; - ins.push_back(in); + ins_.push_back(std::move(in)); } // Add all implicit deps, counting how many as we go. @@ -266,10 +271,10 @@ bool ManifestParser::ParseEdge(string* err) { for (;;) { EvalString in; if (!lexer_.ReadPath(&in, err)) - return err; + return false; if (in.empty()) break; - ins.push_back(in); + ins_.push_back(std::move(in)); ++implicit; } } @@ -283,11 +288,23 @@ bool ManifestParser::ParseEdge(string* err) { return false; if (in.empty()) break; - ins.push_back(in); + ins_.push_back(std::move(in)); ++order_only; } } + // Add all validations, counting how many as we go. + if (lexer_.PeekToken(Lexer::PIPEAT)) { + for (;;) { + EvalString validation; + if (!lexer_.ReadPath(&validation, err)) + return false; + if (validation.empty()) + break; + validations_.push_back(std::move(validation)); + } + } + if (!ExpectToken(Lexer::NEWLINE, err)) return false; @@ -315,30 +332,19 @@ bool ManifestParser::ParseEdge(string* err) { edge->pool_ = pool; } - edge->outputs_.reserve(outs.size()); - for (size_t i = 0, e = outs.size(); i != e; ++i) { - string path = outs[i].Evaluate(env); - string path_err; + edge->outputs_.reserve(outs_.size()); + for (size_t i = 0, e = outs_.size(); i != e; ++i) { + string path = outs_[i].Evaluate(env); + if (path.empty()) + return lexer_.Error("empty path", err); uint64_t slash_bits; - if (!CanonicalizePath(&path, &slash_bits, &path_err)) - return lexer_.Error(path_err, err); - if (!state_->AddOut(edge, path, slash_bits)) { - if (options_.dupe_edge_action_ == kDupeEdgeActionError) { - lexer_.Error("multiple rules generate " + path + " [-w dupbuild=err]", - err); - return false; - } else { - if (!quiet_) { - Warning("multiple rules generate %s. " - "builds involving this target will not be correct; " - "continuing anyway [-w dupbuild=warn]", - path.c_str()); - } - if (e - i <= static_cast(implicit_outs)) - --implicit_outs; - } + CanonicalizePath(&path, &slash_bits); + if (!state_->AddOut(edge, path, slash_bits, err)) { + lexer_.Error(std::string(*err), err); + return false; } } + if (edge->outputs_.empty()) { // All outputs of the edge are already created by other edges. Don't add // this edge. Do this check before input nodes are connected to the edge. @@ -348,18 +354,29 @@ bool ManifestParser::ParseEdge(string* err) { } edge->implicit_outs_ = implicit_outs; - edge->inputs_.reserve(ins.size()); - for (vector::iterator i = ins.begin(); i != ins.end(); ++i) { + edge->inputs_.reserve(ins_.size()); + for (vector::iterator i = ins_.begin(); i != ins_.end(); ++i) { string path = i->Evaluate(env); - string path_err; + if (path.empty()) + return lexer_.Error("empty path", err); uint64_t slash_bits; - if (!CanonicalizePath(&path, &slash_bits, &path_err)) - return lexer_.Error(path_err, err); + CanonicalizePath(&path, &slash_bits); state_->AddIn(edge, path, slash_bits); } edge->implicit_deps_ = implicit; edge->order_only_deps_ = order_only; + edge->validations_.reserve(validations_.size()); + for (std::vector::iterator v = validations_.begin(); + v != validations_.end(); ++v) { + string path = v->Evaluate(env); + if (path.empty()) + return lexer_.Error("empty path", err); + uint64_t slash_bits; + CanonicalizePath(&path, &slash_bits); + state_->AddValidation(edge, path, slash_bits); + } + if (options_.phony_cycle_action_ == kPhonyCycleActionWarn && edge->maybe_phonycycle_diagnostic()) { // CMake 2.8.12.x and 3.0.x incorrectly write phony build statements @@ -379,22 +396,13 @@ bool ManifestParser::ParseEdge(string* err) { } } - // Multiple outputs aren't (yet?) supported with depslog. - string deps_type = edge->GetBinding("deps"); - if (!deps_type.empty() && edge->outputs_.size() > 1) { - return lexer_.Error("multiple outputs aren't (yet?) supported by depslog; " - "bring this up on the mailing list if it affects you", - err); - } - // Lookup, validate, and save any dyndep binding. It will be used later // to load generated dependency information dynamically, but it must // be one of our manifest-specified inputs. string dyndep = edge->GetUnescapedDyndep(); if (!dyndep.empty()) { uint64_t slash_bits; - if (!CanonicalizePath(&dyndep, &slash_bits, err)) - return false; + CanonicalizePath(&dyndep, &slash_bits); edge->dyndep_ = state_->GetNode(dyndep, slash_bits); edge->dyndep_->set_dyndep_pending(true); vector::iterator dgi = @@ -402,6 +410,7 @@ bool ManifestParser::ParseEdge(string* err) { if (dgi == edge->inputs_.end()) { return lexer_.Error("dyndep '" + dyndep + "' is not an input", err); } + assert(!edge->dyndep_->generated_by_dep_loader()); } return true; @@ -413,14 +422,16 @@ bool ManifestParser::ParseFileInclude(bool new_scope, string* err) { return false; string path = eval.Evaluate(env_); - ManifestParser subparser(state_, file_reader_, options_); + if (subparser_ == nullptr) { + subparser_.reset(new ManifestParser(state_, file_reader_, options_)); + } if (new_scope) { - subparser.env_ = new BindingEnv(env_); + subparser_->env_ = new BindingEnv(env_); } else { - subparser.env_ = env_; + subparser_->env_ = env_; } - if (!subparser.Load(path, err, &lexer_)) + if (!subparser_->Load(path, err, &lexer_)) return false; if (!ExpectToken(Lexer::NEWLINE, err)) diff --git a/src/manifest_parser.h b/src/manifest_parser.h index e14d069ea9..ce37759676 100644 --- a/src/manifest_parser.h +++ b/src/manifest_parser.h @@ -17,6 +17,9 @@ #include "parser.h" +#include +#include + struct BindingEnv; struct EvalString; @@ -31,11 +34,7 @@ enum PhonyCycleAction { }; struct ManifestParserOptions { - ManifestParserOptions() - : dupe_edge_action_(kDupeEdgeActionWarn), - phony_cycle_action_(kPhonyCycleActionWarn) {} - DupeEdgeAction dupe_edge_action_; - PhonyCycleAction phony_cycle_action_; + PhonyCycleAction phony_cycle_action_ = kPhonyCycleActionWarn; }; /// Parses .ninja files. @@ -44,28 +43,35 @@ struct ManifestParser : public Parser { ManifestParserOptions options = ManifestParserOptions()); /// Parse a text string of input. Used by tests. - bool ParseTest(const string& input, string* err) { + bool ParseTest(const std::string& input, std::string* err) { quiet_ = true; return Parse("input", input, err); } private: /// Parse a file, given its contents as a string. - bool Parse(const string& filename, const string& input, string* err); + bool Parse(const std::string& filename, const std::string& input, + std::string* err); /// Parse various statement types. - bool ParsePool(string* err); - bool ParseRule(string* err); - bool ParseLet(string* key, EvalString* val, string* err); - bool ParseEdge(string* err); - bool ParseDefault(string* err); + bool ParsePool(std::string* err); + bool ParseRule(std::string* err); + bool ParseLet(std::string* key, EvalString* val, std::string* err); + bool ParseEdge(std::string* err); + bool ParseDefault(std::string* err); /// Parse either a 'subninja' or 'include' line. - bool ParseFileInclude(bool new_scope, string* err); + bool ParseFileInclude(bool new_scope, std::string* err); BindingEnv* env_; ManifestParserOptions options_; bool quiet_; + + // ins_/out_/validations_ are reused across invocations to ParseEdge(), + // to save on the otherwise constant memory reallocation. + // subparser_ is reused solely to get better reuse out ins_/outs_/validation_. + std::unique_ptr subparser_; + std::vector ins_, outs_, validations_; }; #endif // NINJA_MANIFEST_PARSER_H_ diff --git a/src/manifest_parser_perftest.cc b/src/manifest_parser_perftest.cc index 67d11f9166..853d8e0d5d 100644 --- a/src/manifest_parser_perftest.cc +++ b/src/manifest_parser_perftest.cc @@ -25,6 +25,9 @@ #ifdef _WIN32 #include "getopt.h" #include +#elif defined(_AIX) +#include "getopt.h" +#include #else #include #include @@ -37,6 +40,8 @@ #include "state.h" #include "util.h" +using namespace std; + bool WriteFakeManifests(const string& dir, string* err) { RealDiskInterface disk_interface; TimeStamp mtime = disk_interface.Stat(dir + "/build.ninja", err); diff --git a/src/manifest_parser_test.cc b/src/manifest_parser_test.cc index f2b746790a..03ce0b1b80 100644 --- a/src/manifest_parser_test.cc +++ b/src/manifest_parser_test.cc @@ -21,6 +21,8 @@ #include "state.h" #include "test.h" +using namespace std; + struct ParserTest : public testing::Test { void AssertParse(const char* input) { ManifestParser parser(&state, &fs_); @@ -49,7 +51,7 @@ TEST_F(ParserTest, Rules) { "build result: cat in_1.cc in-2.O\n")); ASSERT_EQ(3u, state.bindings_.GetRules().size()); - const Rule* rule = state.bindings_.GetRules().begin()->second; + const auto& rule = state.bindings_.GetRules().begin()->second; EXPECT_EQ("cat", rule->name()); EXPECT_EQ("[cat ][$in][ > ][$out]", rule->GetBinding("command")->Serialize()); @@ -82,7 +84,7 @@ TEST_F(ParserTest, IgnoreIndentedComments) { " #comment\n")); ASSERT_EQ(2u, state.bindings_.GetRules().size()); - const Rule* rule = state.bindings_.GetRules().begin()->second; + const auto& rule = state.bindings_.GetRules().begin()->second; EXPECT_EQ("cat", rule->name()); Edge* edge = state.GetNode("result", 0)->in_edge(); EXPECT_TRUE(edge->GetBindingBool("restat")); @@ -115,7 +117,7 @@ TEST_F(ParserTest, ResponseFiles) { " rspfile=out.rsp\n")); ASSERT_EQ(2u, state.bindings_.GetRules().size()); - const Rule* rule = state.bindings_.GetRules().begin()->second; + const auto& rule = state.bindings_.GetRules().begin()->second; EXPECT_EQ("cat_rsp", rule->name()); EXPECT_EQ("[cat ][$rspfile][ > ][$out]", rule->GetBinding("command")->Serialize()); @@ -132,7 +134,7 @@ TEST_F(ParserTest, InNewline) { " rspfile=out.rsp\n")); ASSERT_EQ(2u, state.bindings_.GetRules().size()); - const Rule* rule = state.bindings_.GetRules().begin()->second; + const auto& rule = state.bindings_.GetRules().begin()->second; EXPECT_EQ("cat_rsp", rule->name()); EXPECT_EQ("[cat ][$in_newline][ > ][$out]", rule->GetBinding("command")->Serialize()); @@ -193,7 +195,7 @@ TEST_F(ParserTest, Continuation) { " d e f\n")); ASSERT_EQ(2u, state.bindings_.GetRules().size()); - const Rule* rule = state.bindings_.GetRules().begin()->second; + const auto& rule = state.bindings_.GetRules().begin()->second; EXPECT_EQ("link", rule->name()); EXPECT_EQ("[foo bar baz]", rule->GetBinding("command")->Serialize()); } @@ -328,29 +330,6 @@ TEST_F(ParserTest, CanonicalizePathsBackslashes) { } #endif -TEST_F(ParserTest, DuplicateEdgeWithMultipleOutputs) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule cat\n" -" command = cat $in > $out\n" -"build out1 out2: cat in1\n" -"build out1: cat in2\n" -"build final: cat out1\n" -)); - // AssertParse() checks that the generated build graph is self-consistent. - // That's all the checking that this test needs. -} - -TEST_F(ParserTest, NoDeadPointerFromDuplicateEdge) { - ASSERT_NO_FATAL_FAILURE(AssertParse( -"rule cat\n" -" command = cat $in > $out\n" -"build out: cat in\n" -"build out: cat in\n" -)); - // AssertParse() checks that the generated build graph is self-consistent. - // That's all the checking that this test needs. -} - TEST_F(ParserTest, DuplicateEdgeWithMultipleOutputsError) { const char kInput[] = "rule cat\n" @@ -358,12 +337,10 @@ TEST_F(ParserTest, DuplicateEdgeWithMultipleOutputsError) { "build out1 out2: cat in1\n" "build out1: cat in2\n" "build final: cat out1\n"; - ManifestParserOptions parser_opts; - parser_opts.dupe_edge_action_ = kDupeEdgeActionError; - ManifestParser parser(&state, &fs_, parser_opts); + ManifestParser parser(&state, &fs_); string err; EXPECT_FALSE(parser.ParseTest(kInput, &err)); - EXPECT_EQ("input:5: multiple rules generate out1 [-w dupbuild=err]\n", err); + EXPECT_EQ("input:5: multiple rules generate out1\n", err); } TEST_F(ParserTest, DuplicateEdgeInIncludedFile) { @@ -375,13 +352,10 @@ TEST_F(ParserTest, DuplicateEdgeInIncludedFile) { "build final: cat out1\n"); const char kInput[] = "subninja sub.ninja\n"; - ManifestParserOptions parser_opts; - parser_opts.dupe_edge_action_ = kDupeEdgeActionError; - ManifestParser parser(&state, &fs_, parser_opts); + ManifestParser parser(&state, &fs_); string err; EXPECT_FALSE(parser.ParseTest(kInput, &err)); - EXPECT_EQ("sub.ninja:5: multiple rules generate out1 [-w dupbuild=err]\n", - err); + EXPECT_EQ("sub.ninja:5: multiple rules generate out1\n", err); } TEST_F(ParserTest, PhonySelfReferenceIgnored) { @@ -406,7 +380,7 @@ TEST_F(ParserTest, PhonySelfReferenceKept) { Node* node = state.LookupNode("a"); Edge* edge = node->in_edge(); - ASSERT_EQ(edge->inputs_.size(), 1); + ASSERT_EQ(edge->inputs_.size(), size_t(1)); ASSERT_EQ(edge->inputs_[0], node); } @@ -858,11 +832,10 @@ TEST_F(ParserTest, MultipleOutputsWithDeps) { State local_state; ManifestParser parser(&local_state, NULL); string err; - EXPECT_FALSE(parser.ParseTest("rule cc\n command = foo\n deps = gcc\n" + EXPECT_TRUE(parser.ParseTest("rule cc\n command = foo\n deps = gcc\n" "build a.o b.o: cc c.cc\n", &err)); - EXPECT_EQ("input:5: multiple outputs aren't (yet?) supported by depslog; " - "bring this up on the mailing list if it affects you\n", err); + EXPECT_EQ("", err); } TEST_F(ParserTest, SubNinja) { @@ -965,6 +938,16 @@ TEST_F(ParserTest, OrderOnly) { ASSERT_TRUE(edge->is_order_only(1)); } +TEST_F(ParserTest, Validations) { + ASSERT_NO_FATAL_FAILURE(AssertParse( +"rule cat\n command = cat $in > $out\n" +"build foo: cat bar |@ baz\n")); + + Edge* edge = state.LookupNode("foo")->in_edge(); + ASSERT_EQ(edge->validations_.size(), size_t(1)); + EXPECT_EQ(edge->validations_[0]->path(), "baz"); +} + TEST_F(ParserTest, ImplicitOutput) { ASSERT_NO_FATAL_FAILURE(AssertParse( "rule cat\n" @@ -972,7 +955,7 @@ TEST_F(ParserTest, ImplicitOutput) { "build foo | imp: cat bar\n")); Edge* edge = state.LookupNode("imp")->in_edge(); - ASSERT_EQ(edge->outputs_.size(), 2); + ASSERT_EQ(edge->outputs_.size(), size_t(2)); EXPECT_TRUE(edge->is_implicit_out(1)); } @@ -983,32 +966,30 @@ TEST_F(ParserTest, ImplicitOutputEmpty) { "build foo | : cat bar\n")); Edge* edge = state.LookupNode("foo")->in_edge(); - ASSERT_EQ(edge->outputs_.size(), 1); + ASSERT_EQ(edge->outputs_.size(), size_t(1)); EXPECT_FALSE(edge->is_implicit_out(0)); } -TEST_F(ParserTest, ImplicitOutputDupe) { - ASSERT_NO_FATAL_FAILURE(AssertParse( +TEST_F(ParserTest, ImplicitOutputDupeError) { + const char kInput[] = "rule cat\n" " command = cat $in > $out\n" -"build foo baz | foo baq foo: cat bar\n")); - - Edge* edge = state.LookupNode("foo")->in_edge(); - ASSERT_EQ(edge->outputs_.size(), 3); - EXPECT_FALSE(edge->is_implicit_out(0)); - EXPECT_FALSE(edge->is_implicit_out(1)); - EXPECT_TRUE(edge->is_implicit_out(2)); +"build foo baz | foo baq foo: cat bar\n"; + ManifestParser parser(&state, &fs_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:4: foo is defined as an output multiple times\n", err); } -TEST_F(ParserTest, ImplicitOutputDupes) { - ASSERT_NO_FATAL_FAILURE(AssertParse( +TEST_F(ParserTest, ImplicitOutputDupesError) { + const char kInput[] = "rule cat\n" " command = cat $in > $out\n" -"build foo foo foo | foo foo foo foo: cat bar\n")); - - Edge* edge = state.LookupNode("foo")->in_edge(); - ASSERT_EQ(edge->outputs_.size(), 1); - EXPECT_FALSE(edge->is_implicit_out(0)); +"build foo foo foo | foo foo foo foo: cat bar\n"; + ManifestParser parser(&state, &fs_); + string err; + EXPECT_FALSE(parser.ParseTest(kInput, &err)); + EXPECT_EQ("input:4: foo is defined as an output multiple times\n", err); } TEST_F(ParserTest, NoExplicitOutput) { diff --git a/src/metrics.cc b/src/metrics.cc index a7d3c7ad5b..e7cb4d1444 100644 --- a/src/metrics.cc +++ b/src/metrics.cc @@ -18,63 +18,42 @@ #include #include -#ifndef _WIN32 -#include -#else -#include -#endif - #include +#include #include "util.h" +using namespace std; + Metrics* g_metrics = NULL; namespace { -#ifndef _WIN32 /// Compute a platform-specific high-res timer value that fits into an int64. int64_t HighResTimer() { - timeval tv; - if (gettimeofday(&tv, NULL) < 0) - Fatal("gettimeofday: %s", strerror(errno)); - return (int64_t)tv.tv_sec * 1000*1000 + tv.tv_usec; + auto now = chrono::steady_clock::now(); + return chrono::duration_cast( + now.time_since_epoch()) + .count(); } -/// Convert a delta of HighResTimer() values to microseconds. int64_t TimerToMicros(int64_t dt) { - // No conversion necessary. - return dt; -} -#else -int64_t LargeIntegerToInt64(const LARGE_INTEGER& i) { - return ((int64_t)i.HighPart) << 32 | i.LowPart; -} - -int64_t HighResTimer() { - LARGE_INTEGER counter; - if (!QueryPerformanceCounter(&counter)) - Fatal("QueryPerformanceCounter: %s", GetLastErrorString().c_str()); - return LargeIntegerToInt64(counter); + // dt is in ticks. We want microseconds. + return chrono::duration_cast( + std::chrono::steady_clock::duration{ dt }) + .count(); } -int64_t TimerToMicros(int64_t dt) { - static int64_t ticks_per_sec = 0; - if (!ticks_per_sec) { - LARGE_INTEGER freq; - if (!QueryPerformanceFrequency(&freq)) - Fatal("QueryPerformanceFrequency: %s", GetLastErrorString().c_str()); - ticks_per_sec = LargeIntegerToInt64(freq); - } - +int64_t TimerToMicros(double dt) { // dt is in ticks. We want microseconds. - return (dt * 1000000) / ticks_per_sec; + using DoubleSteadyClock = + std::chrono::duration; + return chrono::duration_cast(DoubleSteadyClock{ dt }) + .count(); } -#endif } // anonymous namespace - ScopedMetric::ScopedMetric(Metric* metric) { metric_ = metric; if (!metric_) @@ -85,7 +64,9 @@ ScopedMetric::~ScopedMetric() { if (!metric_) return; metric_->count++; - int64_t dt = TimerToMicros(HighResTimer() - start_); + // Leave in the timer's natural frequency to avoid paying the conversion cost + // on every measurement. + int64_t dt = HighResTimer() - start_; metric_->sum += dt; } @@ -110,18 +91,23 @@ void Metrics::Report() { for (vector::iterator i = metrics_.begin(); i != metrics_.end(); ++i) { Metric* metric = *i; - double total = metric->sum / (double)1000; - double avg = metric->sum / (double)metric->count; + uint64_t micros = TimerToMicros(metric->sum); + double total = micros / (double)1000; + double avg = micros / (double)metric->count; printf("%-*s\t%-6d\t%-8.1f\t%.1f\n", width, metric->name.c_str(), metric->count, avg, total); } } -uint64_t Stopwatch::Now() const { - return TimerToMicros(HighResTimer()); +double Stopwatch::Elapsed() const { + // Convert to micros after converting to double to minimize error. + return 1e-6 * TimerToMicros(static_cast(NowRaw() - started_)); +} + +uint64_t Stopwatch::NowRaw() const { + return HighResTimer(); } int64_t GetTimeMillis() { return TimerToMicros(HighResTimer()) / 1000; } - diff --git a/src/metrics.h b/src/metrics.h index b6da859db2..937d905de7 100644 --- a/src/metrics.h +++ b/src/metrics.h @@ -17,7 +17,6 @@ #include #include -using namespace std; #include "util.h" // For int64_t. @@ -26,14 +25,13 @@ using namespace std; /// A single metrics we're tracking, like "depfile load time". struct Metric { - string name; + std::string name; /// Number of times we've hit the code path. int count; - /// Total time (in micros) we've spent on the code path. + /// Total time (in platform-dependent units) we've spent on the code path. int64_t sum; }; - /// A scoped object for recording a metric across the body of a function. /// Used by the METRIC_RECORD macro. struct ScopedMetric { @@ -49,13 +47,13 @@ struct ScopedMetric { /// The singleton that stores metrics and prints the report. struct Metrics { - Metric* NewMetric(const string& name); + Metric* NewMetric(const std::string& name); /// Print a summary report to stdout. void Report(); private: - vector metrics_; + std::vector metrics_; }; /// Get the current time as relative to some epoch. @@ -69,15 +67,15 @@ struct Stopwatch { Stopwatch() : started_(0) {} /// Seconds since Restart() call. - double Elapsed() const { - return 1e-6 * static_cast(Now() - started_); - } + double Elapsed() const; - void Restart() { started_ = Now(); } + void Restart() { started_ = NowRaw(); } private: uint64_t started_; - uint64_t Now() const; + // Return the current time using the native frequency of the high resolution + // timer. + uint64_t NowRaw() const; }; /// The primary interface to metrics. Use METRIC_RECORD("foobar") at the top @@ -87,6 +85,13 @@ struct Stopwatch { g_metrics ? g_metrics->NewMetric(name) : NULL; \ ScopedMetric metrics_h_scoped(metrics_h_metric); +/// A variant of METRIC_RECORD that doesn't record anything if |condition| +/// is false. +#define METRIC_RECORD_IF(name, condition) \ + static Metric* metrics_h_metric = \ + g_metrics ? g_metrics->NewMetric(name) : NULL; \ + ScopedMetric metrics_h_scoped((condition) ? metrics_h_metric : NULL); + extern Metrics* g_metrics; #endif // NINJA_METRICS_H_ diff --git a/src/minidump-win32.cc b/src/minidump-win32.cc index ca936387bd..c0d27a53e7 100644 --- a/src/minidump-win32.cc +++ b/src/minidump-win32.cc @@ -15,10 +15,12 @@ #ifdef _MSC_VER #include -#include +#include #include "util.h" +using namespace std; + typedef BOOL (WINAPI *MiniDumpWriteDumpFunc) ( IN HANDLE, IN DWORD, @@ -49,8 +51,8 @@ void CreateWin32MiniDump(_EXCEPTION_POINTERS* pep) { return; } - MiniDumpWriteDumpFunc mini_dump_write_dump = - (MiniDumpWriteDumpFunc)GetProcAddress(dbghelp, "MiniDumpWriteDump"); + MiniDumpWriteDumpFunc mini_dump_write_dump = FunctionCast + (GetProcAddress(dbghelp, "MiniDumpWriteDump")); if (mini_dump_write_dump == NULL) { Error("failed to create minidump: GetProcAddress('MiniDumpWriteDump'): %s", GetLastErrorString().c_str()); diff --git a/src/missing_deps.cc b/src/missing_deps.cc new file mode 100644 index 0000000000..f96a5e7b29 --- /dev/null +++ b/src/missing_deps.cc @@ -0,0 +1,193 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "missing_deps.h" + +#include + +#include + +#include "depfile_parser.h" +#include "deps_log.h" +#include "disk_interface.h" +#include "graph.h" +#include "state.h" +#include "util.h" + +namespace { + +/// ImplicitDepLoader variant that stores dep nodes into the given output +/// without updating graph deps like the base loader does. +struct NodeStoringImplicitDepLoader : public ImplicitDepLoader { + NodeStoringImplicitDepLoader( + State* state, DepsLog* deps_log, DiskInterface* disk_interface, + DepfileParserOptions const* depfile_parser_options, + Explanations* explanations, std::vector* dep_nodes_output) + : ImplicitDepLoader(state, deps_log, disk_interface, + depfile_parser_options, explanations), + dep_nodes_output_(dep_nodes_output) {} + + protected: + virtual bool ProcessDepfileDeps(Edge* edge, + std::vector* depfile_ins, + std::string* err); + + private: + std::vector* dep_nodes_output_; +}; + +bool NodeStoringImplicitDepLoader::ProcessDepfileDeps( + Edge* edge, std::vector* depfile_ins, std::string* err) { + for (std::vector::iterator i = depfile_ins->begin(); + i != depfile_ins->end(); ++i) { + uint64_t slash_bits; + CanonicalizePath(const_cast(i->str_), &i->len_, &slash_bits); + Node* node = state_->GetNode(*i, slash_bits); + dep_nodes_output_->push_back(node); + } + return true; +} + +} // namespace + +MissingDependencyScannerDelegate::~MissingDependencyScannerDelegate() {} + +void MissingDependencyPrinter::OnMissingDep(Node* node, const std::string& path, + const Rule& generator) { + std::cout << "Missing dep: " << node->path() << " uses " << path + << " (generated by " << generator.name() << ")\n"; +} + +MissingDependencyScanner::MissingDependencyScanner( + MissingDependencyScannerDelegate* delegate, DepsLog* deps_log, State* state, + DiskInterface* disk_interface) + : delegate_(delegate), deps_log_(deps_log), state_(state), + disk_interface_(disk_interface), missing_dep_path_count_(0) {} + +void MissingDependencyScanner::ProcessNode(Node* node) { + if (!node) + return; + Edge* edge = node->in_edge(); + if (!edge) + return; + if (!seen_.insert(node).second) + return; + + for (std::vector::iterator in = edge->inputs_.begin(); + in != edge->inputs_.end(); ++in) { + ProcessNode(*in); + } + + std::string deps_type = edge->GetBinding("deps"); + if (!deps_type.empty()) { + DepsLog::Deps* deps = deps_log_->GetDeps(node); + if (deps) + ProcessNodeDeps(node, deps->nodes, deps->node_count); + } else { + DepfileParserOptions parser_opts; + std::vector depfile_deps; + NodeStoringImplicitDepLoader dep_loader(state_, deps_log_, disk_interface_, + &parser_opts, nullptr, + &depfile_deps); + std::string err; + dep_loader.LoadDeps(edge, &err); + if (!depfile_deps.empty()) + ProcessNodeDeps(node, &depfile_deps[0], depfile_deps.size()); + } +} + +void MissingDependencyScanner::ProcessNodeDeps(Node* node, Node** dep_nodes, + int dep_nodes_count) { + Edge* edge = node->in_edge(); + std::set deplog_edges; + for (int i = 0; i < dep_nodes_count; ++i) { + Node* deplog_node = dep_nodes[i]; + // Special exception: A dep on build.ninja can be used to mean "always + // rebuild this target when the build is reconfigured", but build.ninja is + // often generated by a configuration tool like cmake or gn. The rest of + // the build "implicitly" depends on the entire build being reconfigured, + // so a missing dep path to build.ninja is not an actual missing dependency + // problem. + if (deplog_node->path() == "build.ninja") + return; + Edge* deplog_edge = deplog_node->in_edge(); + if (deplog_edge) { + deplog_edges.insert(deplog_edge); + } + } + std::vector missing_deps; + for (std::set::iterator de = deplog_edges.begin(); + de != deplog_edges.end(); ++de) { + if (!PathExistsBetween(*de, edge)) { + missing_deps.push_back(*de); + } + } + + if (!missing_deps.empty()) { + std::set missing_deps_rule_names; + for (std::vector::iterator ne = missing_deps.begin(); + ne != missing_deps.end(); ++ne) { + for (int i = 0; i < dep_nodes_count; ++i) { + if (dep_nodes[i]->in_edge() == *ne) { + generated_nodes_.insert(dep_nodes[i]); + generator_rules_.insert(&(*ne)->rule()); + missing_deps_rule_names.insert((*ne)->rule().name()); + delegate_->OnMissingDep(node, dep_nodes[i]->path(), (*ne)->rule()); + } + } + } + missing_dep_path_count_ += missing_deps_rule_names.size(); + nodes_missing_deps_.insert(node); + } +} + +void MissingDependencyScanner::PrintStats() { + std::cout << "Processed " << seen_.size() << " nodes.\n"; + if (HadMissingDeps()) { + std::cout << "Error: There are " << missing_dep_path_count_ + << " missing dependency paths.\n"; + std::cout << nodes_missing_deps_.size() + << " targets had depfile dependencies on " + << generated_nodes_.size() << " distinct generated inputs " + << "(from " << generator_rules_.size() << " rules) " + << " without a non-depfile dep path to the generator.\n"; + std::cout << "There might be build flakiness if any of the targets listed " + "above are built alone, or not late enough, in a clean output " + "directory.\n"; + } else { + std::cout << "No missing dependencies on generated files found.\n"; + } +} + +bool MissingDependencyScanner::PathExistsBetween(Edge* from, Edge* to) { + AdjacencyMap::iterator it = adjacency_map_.find(from); + if (it != adjacency_map_.end()) { + InnerAdjacencyMap::iterator inner_it = it->second.find(to); + if (inner_it != it->second.end()) { + return inner_it->second; + } + } else { + it = adjacency_map_.insert(std::make_pair(from, InnerAdjacencyMap())).first; + } + bool found = false; + for (size_t i = 0; i < to->inputs_.size(); ++i) { + Edge* e = to->inputs_[i]->in_edge(); + if (e && (e == from || PathExistsBetween(from, e))) { + found = true; + break; + } + } + it->second.insert(std::make_pair(to, found)); + return found; +} diff --git a/src/missing_deps.h b/src/missing_deps.h new file mode 100644 index 0000000000..7a615da2a5 --- /dev/null +++ b/src/missing_deps.h @@ -0,0 +1,74 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_MISSING_DEPS_H_ +#define NINJA_MISSING_DEPS_H_ + +#include +#include +#include + +#include + +struct DepsLog; +struct DiskInterface; +struct Edge; +struct Node; +struct Rule; +struct State; + +class MissingDependencyScannerDelegate { + public: + virtual ~MissingDependencyScannerDelegate(); + virtual void OnMissingDep(Node* node, const std::string& path, + const Rule& generator) = 0; +}; + +class MissingDependencyPrinter : public MissingDependencyScannerDelegate { + void OnMissingDep(Node* node, const std::string& path, const Rule& generator); + void OnStats(int nodes_processed, int nodes_missing_deps, + int missing_dep_path_count, int generated_nodes, + int generator_rules); +}; + +struct MissingDependencyScanner { + public: + MissingDependencyScanner(MissingDependencyScannerDelegate* delegate, + DepsLog* deps_log, State* state, + DiskInterface* disk_interface); + void ProcessNode(Node* node); + void PrintStats(); + bool HadMissingDeps() { return !nodes_missing_deps_.empty(); } + + void ProcessNodeDeps(Node* node, Node** dep_nodes, int dep_nodes_count); + + bool PathExistsBetween(Edge* from, Edge* to); + + MissingDependencyScannerDelegate* delegate_; + DepsLog* deps_log_; + State* state_; + DiskInterface* disk_interface_; + std::set seen_; + std::set nodes_missing_deps_; + std::set generated_nodes_; + std::set generator_rules_; + int missing_dep_path_count_; + + private: + using InnerAdjacencyMap = std::unordered_map; + using AdjacencyMap = std::unordered_map; + AdjacencyMap adjacency_map_; +}; + +#endif // NINJA_MISSING_DEPS_H_ diff --git a/src/missing_deps_test.cc b/src/missing_deps_test.cc new file mode 100644 index 0000000000..dae377b49d --- /dev/null +++ b/src/missing_deps_test.cc @@ -0,0 +1,167 @@ +// Copyright 2019 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +#include "deps_log.h" +#include "graph.h" +#include "missing_deps.h" +#include "state.h" +#include "test.h" + +const char kTestDepsLogFilename[] = "MissingDepTest-tempdepslog"; + +class MissingDependencyTestDelegate : public MissingDependencyScannerDelegate { + void OnMissingDep(Node* node, const std::string& path, + const Rule& generator) {} +}; + +struct MissingDependencyScannerTest : public testing::Test { + MissingDependencyScannerTest() + : generator_rule_("generator_rule"), compile_rule_("compile_rule"), + scanner_(&delegate_, &deps_log_, &state_, &filesystem_) { + std::string err; + deps_log_.OpenForWrite(kTestDepsLogFilename, &err); + EXPECT_EQ("", err); + } + + ~MissingDependencyScannerTest() { + // Remove test file. + deps_log_.Close(); + } + + MissingDependencyScanner& scanner() { return scanner_; } + + void RecordDepsLogDep(const std::string& from, const std::string& to) { + Node* node_deps[] = { state_.LookupNode(to) }; + deps_log_.RecordDeps(state_.LookupNode(from), 0, 1, node_deps); + } + + void ProcessAllNodes() { + std::string err; + std::vector nodes = state_.RootNodes(&err); + EXPECT_EQ("", err); + for (std::vector::iterator it = nodes.begin(); it != nodes.end(); + ++it) { + scanner().ProcessNode(*it); + } + } + + void CreateInitialState() { + EvalString deps_type; + deps_type.AddText("gcc"); + compile_rule_.AddBinding("deps", deps_type); + generator_rule_.AddBinding("deps", deps_type); + Edge* header_edge = state_.AddEdge(&generator_rule_); + state_.AddOut(header_edge, "generated_header", 0, nullptr); + Edge* compile_edge = state_.AddEdge(&compile_rule_); + state_.AddOut(compile_edge, "compiled_object", 0, nullptr); + } + + void CreateGraphDependencyBetween(const char* from, const char* to) { + Node* from_node = state_.LookupNode(from); + Edge* from_edge = from_node->in_edge(); + state_.AddIn(from_edge, to, 0); + } + + void AssertMissingDependencyBetween(const char* flaky, const char* generated, + Rule* rule) { + Node* flaky_node = state_.LookupNode(flaky); + ASSERT_EQ(1u, scanner().nodes_missing_deps_.count(flaky_node)); + Node* generated_node = state_.LookupNode(generated); + ASSERT_EQ(1u, scanner().generated_nodes_.count(generated_node)); + ASSERT_EQ(1u, scanner().generator_rules_.count(rule)); + } + + ScopedFilePath scoped_file_path_ = kTestDepsLogFilename; + MissingDependencyTestDelegate delegate_; + Rule generator_rule_; + Rule compile_rule_; + DepsLog deps_log_; + State state_; + VirtualFileSystem filesystem_; + MissingDependencyScanner scanner_; +}; + +TEST_F(MissingDependencyScannerTest, EmptyGraph) { + ProcessAllNodes(); + ASSERT_FALSE(scanner().HadMissingDeps()); +} + +TEST_F(MissingDependencyScannerTest, NoMissingDep) { + CreateInitialState(); + ProcessAllNodes(); + ASSERT_FALSE(scanner().HadMissingDeps()); +} + +TEST_F(MissingDependencyScannerTest, MissingDepPresent) { + CreateInitialState(); + // compiled_object uses generated_header, without a proper dependency + RecordDepsLogDep("compiled_object", "generated_header"); + ProcessAllNodes(); + ASSERT_TRUE(scanner().HadMissingDeps()); + ASSERT_EQ(1u, scanner().nodes_missing_deps_.size()); + ASSERT_EQ(1u, scanner().missing_dep_path_count_); + AssertMissingDependencyBetween("compiled_object", "generated_header", + &generator_rule_); +} + +TEST_F(MissingDependencyScannerTest, MissingDepFixedDirect) { + CreateInitialState(); + // Adding the direct dependency fixes the missing dep + CreateGraphDependencyBetween("compiled_object", "generated_header"); + RecordDepsLogDep("compiled_object", "generated_header"); + ProcessAllNodes(); + ASSERT_FALSE(scanner().HadMissingDeps()); +} + +TEST_F(MissingDependencyScannerTest, MissingDepFixedIndirect) { + CreateInitialState(); + // Adding an indirect dependency also fixes the issue + Edge* intermediate_edge = state_.AddEdge(&generator_rule_); + state_.AddOut(intermediate_edge, "intermediate", 0, nullptr); + CreateGraphDependencyBetween("compiled_object", "intermediate"); + CreateGraphDependencyBetween("intermediate", "generated_header"); + RecordDepsLogDep("compiled_object", "generated_header"); + ProcessAllNodes(); + ASSERT_FALSE(scanner().HadMissingDeps()); +} + +TEST_F(MissingDependencyScannerTest, CyclicMissingDep) { + CreateInitialState(); + RecordDepsLogDep("generated_header", "compiled_object"); + RecordDepsLogDep("compiled_object", "generated_header"); + // In case of a cycle, both paths are reported (and there is + // no way to fix the issue by adding deps). + ProcessAllNodes(); + ASSERT_TRUE(scanner().HadMissingDeps()); + ASSERT_EQ(2u, scanner().nodes_missing_deps_.size()); + ASSERT_EQ(2u, scanner().missing_dep_path_count_); + AssertMissingDependencyBetween("compiled_object", "generated_header", + &generator_rule_); + AssertMissingDependencyBetween("generated_header", "compiled_object", + &compile_rule_); +} + +TEST_F(MissingDependencyScannerTest, CycleInGraph) { + CreateInitialState(); + CreateGraphDependencyBetween("compiled_object", "generated_header"); + CreateGraphDependencyBetween("generated_header", "compiled_object"); + // The missing-deps tool doesn't deal with cycles in the graph, because + // there will be an error loading the graph before we get to the tool. + // This test is to illustrate that. + std::string err; + std::vector nodes = state_.RootNodes(&err); + ASSERT_NE("", err); +} diff --git a/src/msvc_helper-win32.cc b/src/msvc_helper-win32.cc index de6147a5e5..1148ae52a5 100644 --- a/src/msvc_helper-win32.cc +++ b/src/msvc_helper-win32.cc @@ -18,6 +18,8 @@ #include "util.h" +using namespace std; + namespace { string Replace(const string& input, const string& find, const string& replace) { diff --git a/src/msvc_helper.h b/src/msvc_helper.h index 70d1fff794..699b0a132d 100644 --- a/src/msvc_helper.h +++ b/src/msvc_helper.h @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#ifndef MSVC_HELPER_H_ +#define MSVC_HELPER_H_ + #include -using namespace std; -string EscapeForDepfile(const string& path); +std::string EscapeForDepfile(const std::string& path); /// Wraps a synchronous execution of a CL subprocess. struct CLWrapper { @@ -27,7 +29,9 @@ struct CLWrapper { /// Start a process and gather its raw output. Returns its exit code. /// Crashes (calls Fatal()) on error. - int Run(const string& command, string* output); + int Run(const std::string& command, std::string* output); void* env_block_; }; + +#endif // MSVC_HELPER_H_ diff --git a/src/msvc_helper_main-win32.cc b/src/msvc_helper_main-win32.cc index 644b2a2e24..972982c642 100644 --- a/src/msvc_helper_main-win32.cc +++ b/src/msvc_helper_main-win32.cc @@ -24,6 +24,8 @@ #include "getopt.h" +using namespace std; + namespace { void Usage() { @@ -52,23 +54,23 @@ void WriteDepFileOrDie(const char* object_path, const CLParser& parse) { string depfile_path = string(object_path) + ".d"; FILE* depfile = fopen(depfile_path.c_str(), "w"); if (!depfile) { - unlink(object_path); + platformAwareUnlink(object_path); Fatal("opening %s: %s", depfile_path.c_str(), GetLastErrorString().c_str()); } if (fprintf(depfile, "%s: ", object_path) < 0) { - unlink(object_path); + platformAwareUnlink(object_path); fclose(depfile); - unlink(depfile_path.c_str()); + platformAwareUnlink(depfile_path.c_str()); Fatal("writing %s", depfile_path.c_str()); } const set& headers = parse.includes_; for (set::const_iterator i = headers.begin(); i != headers.end(); ++i) { if (fprintf(depfile, "%s\n", EscapeForDepfile(*i).c_str()) < 0) { - unlink(object_path); + platformAwareUnlink(object_path); fclose(depfile); - unlink(depfile_path.c_str()); + platformAwareUnlink(depfile_path.c_str()); Fatal("writing %s", depfile_path.c_str()); } } diff --git a/src/msvc_helper_test.cc b/src/msvc_helper_test.cc index eaae51f564..d9e2ee6623 100644 --- a/src/msvc_helper_test.cc +++ b/src/msvc_helper_test.cc @@ -17,6 +17,8 @@ #include "test.h" #include "util.h" +using namespace std; + TEST(EscapeForDepfileTest, SpacesInFilename) { ASSERT_EQ("sub\\some\\ sdk\\foo.h", EscapeForDepfile("sub\\some sdk\\foo.h")); diff --git a/src/ninja.cc b/src/ninja.cc index a093cd1818..1e934ea678 100644 --- a/src/ninja.cc +++ b/src/ninja.cc @@ -18,6 +18,11 @@ #include #include +#include +#include +#include +#include + #ifdef _WIN32 #include "getopt.h" #include @@ -35,17 +40,24 @@ #include "build_log.h" #include "deps_log.h" #include "clean.h" +#include "command_collector.h" #include "debug_flags.h" #include "disk_interface.h" +#include "exit_status.h" #include "graph.h" #include "graphviz.h" +#include "json.h" #include "manifest_parser.h" #include "metrics.h" +#include "missing_deps.h" #include "state.h" +#include "status.h" #include "util.h" #include "version.h" -#ifdef _MSC_VER +using namespace std; + +#ifdef _WIN32 // Defined in msvc_helper_main-win32.cc. int MSVCHelperMain(int argc, char** argv); @@ -68,22 +80,16 @@ struct Options { /// Tool to run rather than building. const Tool* tool; - /// Whether duplicate rules for one target should warn or print an error. - bool dupe_edges_should_err; - /// Whether phony cycles should warn or print an error. bool phony_cycle_should_err; - - /// Whether a depfile with multiple targets on separate lines should - /// warn or print an error. - bool depfile_distinct_target_lines_should_err; }; /// The Ninja main() loads up a series of data structures; various tools need /// to poke into these, so store them as fields on an object. struct NinjaMain : public BuildLogUser { NinjaMain(const char* ninja_command, const BuildConfig& config) : - ninja_command_(ninja_command), config_(config) {} + ninja_command_(ninja_command), config_(config), + start_time_millis_(GetTimeMillis()) {} /// Command line used to run Ninja. const char* ninja_command_; @@ -94,7 +100,7 @@ struct NinjaMain : public BuildLogUser { /// Loaded state (rules, nodes). State state_; - /// Functions for accesssing the disk. + /// Functions for accessing the disk. RealDiskInterface disk_interface_; /// The build directory, used for storing the build log etc. @@ -118,15 +124,23 @@ struct NinjaMain : public BuildLogUser { int ToolGraph(const Options* options, int argc, char* argv[]); int ToolQuery(const Options* options, int argc, char* argv[]); int ToolDeps(const Options* options, int argc, char* argv[]); + int ToolMissingDeps(const Options* options, int argc, char* argv[]); int ToolBrowse(const Options* options, int argc, char* argv[]); int ToolMSVC(const Options* options, int argc, char* argv[]); int ToolTargets(const Options* options, int argc, char* argv[]); int ToolCommands(const Options* options, int argc, char* argv[]); + int ToolInputs(const Options* options, int argc, char* argv[]); + int ToolMultiInputs(const Options* options, int argc, char* argv[]); int ToolClean(const Options* options, int argc, char* argv[]); + int ToolCleanDead(const Options* options, int argc, char* argv[]); int ToolCompilationDatabase(const Options* options, int argc, char* argv[]); + int ToolCompilationDatabaseForTargets(const Options* options, int argc, + char* argv[]); int ToolRecompact(const Options* options, int argc, char* argv[]); + int ToolRestat(const Options* options, int argc, char* argv[]); int ToolUrtle(const Options* options, int argc, char** argv); int ToolRules(const Options* options, int argc, char* argv[]); + int ToolWinCodePage(const Options* options, int argc, char* argv[]); /// Open the build log. /// @return false on error. @@ -143,18 +157,22 @@ struct NinjaMain : public BuildLogUser { /// Rebuild the manifest, if necessary. /// Fills in \a err on error. /// @return true if the manifest was rebuilt. - bool RebuildManifest(const char* input_file, string* err); + bool RebuildManifest(const char* input_file, string* err, Status* status); + + /// For each edge, lookup in build log how long it took last time, + /// and record that in the edge itself. It will be used for ETA prediction. + void ParsePreviousElapsedTimes(); /// Build the targets listed on the command line. /// @return an exit code. - int RunBuild(int argc, char** argv); + ExitStatus RunBuild(int argc, char** argv, Status* status); /// Dump the output requested by '-d stats'. void DumpMetrics(); virtual bool IsPathDead(StringPiece s) const { Node* n = state_.LookupNode(s); - if (!n || !n->in_edge()) + if (n && n->in_edge()) return false; // Just checking n isn't enough: If an old output is both in the build log // and in the deps log, it will have a Node object in state_. (It will also @@ -171,6 +189,8 @@ struct NinjaMain : public BuildLogUser { Error("%s", err.c_str()); // Log and ignore Stat() errors. return mtime == 0; } + + int64_t start_time_millis_; }; /// Subtools, accessible via "-t foo". @@ -208,6 +228,7 @@ void Usage(const BuildConfig& config) { "options:\n" " --version print ninja version (\"%s\")\n" " -v, --verbose show all command lines while building\n" +" --quiet don't show progress status, just command output\n" "\n" " -C DIR change to DIR before doing anything else\n" " -f FILE specify input build file [default=build.ninja]\n" @@ -239,23 +260,28 @@ int GuessParallelism() { /// Rebuild the build manifest, if necessary. /// Returns true if the manifest was rebuilt. -bool NinjaMain::RebuildManifest(const char* input_file, string* err) { +bool NinjaMain::RebuildManifest(const char* input_file, string* err, + Status* status) { string path = input_file; - uint64_t slash_bits; // Unused because this path is only used for lookup. - if (!CanonicalizePath(&path, &slash_bits, err)) + if (path.empty()) { + *err = "empty path"; return false; + } + uint64_t slash_bits; // Unused because this path is only used for lookup. + CanonicalizePath(&path, &slash_bits); Node* node = state_.LookupNode(path); if (!node) return false; - Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_); + Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_, + status, start_time_millis_); if (!builder.AddTarget(node, err)) return false; if (builder.AlreadyUpToDate()) return false; // Not an error, but we didn't rebuild. - if (!builder.Build(err)) + if (builder.Build(err) != ExitSuccess) return false; // The manifest was only rebuilt if it is now dirty (it may have been cleaned @@ -270,11 +296,27 @@ bool NinjaMain::RebuildManifest(const char* input_file, string* err) { return true; } +void NinjaMain::ParsePreviousElapsedTimes() { + for (Edge* edge : state_.edges_) { + for (Node* out : edge->outputs_) { + BuildLog::LogEntry* log_entry = build_log_.LookupByOutput(out->path()); + if (!log_entry) + continue; // Maybe we'll have log entry for next output of this edge? + edge->prev_elapsed_time_millis = + log_entry->end_time - log_entry->start_time; + break; // Onto next edge. + } + } +} + Node* NinjaMain::CollectTarget(const char* cpath, string* err) { string path = cpath; - uint64_t slash_bits; - if (!CanonicalizePath(&path, &slash_bits, err)) + if (path.empty()) { + *err = "empty path"; return NULL; + } + uint64_t slash_bits; + CanonicalizePath(&path, &slash_bits); // Special syntax: "foo.cc^" means "the first output of foo.cc". bool first_dependent = false; @@ -287,15 +329,20 @@ Node* NinjaMain::CollectTarget(const char* cpath, string* err) { if (node) { if (first_dependent) { if (node->out_edges().empty()) { - *err = "'" + path + "' has no out edge"; - return NULL; - } - Edge* edge = node->out_edges()[0]; - if (edge->outputs_.empty()) { - edge->Dump(); - Fatal("edge has no outputs"); + Node* rev_deps = deps_log_.GetFirstReverseDepsNode(node); + if (!rev_deps) { + *err = "'" + path + "' has no out edge"; + return NULL; + } + node = rev_deps; + } else { + Edge* edge = node->out_edges()[0]; + if (edge->outputs_.empty()) { + edge->Dump(); + Fatal("edge has no outputs"); + } + node = edge->outputs_[0]; } - node = edge->outputs_[0]; } return node; } else { @@ -380,6 +427,13 @@ int NinjaMain::ToolQuery(const Options* options, int argc, char* argv[]) { label = "|| "; printf(" %s%s\n", label, edge->inputs_[in]->path().c_str()); } + if (!edge->validations_.empty()) { + printf(" validations:\n"); + for (std::vector::iterator validation = edge->validations_.begin(); + validation != edge->validations_.end(); ++validation) { + printf(" %s\n", (*validation)->path().c_str()); + } + } } printf(" outputs:\n"); for (vector::const_iterator edge = node->out_edges().begin(); @@ -389,6 +443,17 @@ int NinjaMain::ToolQuery(const Options* options, int argc, char* argv[]) { printf(" %s\n", (*out)->path().c_str()); } } + const std::vector validation_edges = node->validation_out_edges(); + if (!validation_edges.empty()) { + printf(" validation for:\n"); + for (std::vector::const_iterator edge = validation_edges.begin(); + edge != validation_edges.end(); ++edge) { + for (vector::iterator out = (*edge)->outputs_.begin(); + out != (*edge)->outputs_.end(); ++out) { + printf(" %s\n", (*out)->path().c_str()); + } + } + } } return 0; } @@ -406,7 +471,7 @@ int NinjaMain::ToolBrowse(const Options*, int, char**) { } #endif -#if defined(_MSC_VER) +#if defined(_WIN32) int NinjaMain::ToolMSVC(const Options* options, int argc, char* argv[]) { // Reset getopt: push one argument onto the front of argv, reset optind. argc++; @@ -487,7 +552,7 @@ int NinjaMain::ToolDeps(const Options* options, int argc, char** argv) { if (argc == 0) { for (vector::const_iterator ni = deps_log_.nodes().begin(); ni != deps_log_.nodes().end(); ++ni) { - if (deps_log_.IsDepsEntryLiveFor(*ni)) + if (DepsLog::IsDepsEntryLiveFor(*ni)) nodes.push_back(*ni); } } else { @@ -522,6 +587,26 @@ int NinjaMain::ToolDeps(const Options* options, int argc, char** argv) { return 0; } +int NinjaMain::ToolMissingDeps(const Options* options, int argc, char** argv) { + vector nodes; + string err; + if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) { + Error("%s", err.c_str()); + return 1; + } + RealDiskInterface disk_interface; + MissingDependencyPrinter printer; + MissingDependencyScanner scanner(&printer, &deps_log_, &state_, + &disk_interface); + for (vector::iterator it = nodes.begin(); it != nodes.end(); ++it) { + scanner.ProcessNode(*it); + } + scanner.PrintStats(); + if (scanner.HadMissingDeps()) + return 3; + return 0; +} + int NinjaMain::ToolTargets(const Options* options, int argc, char* argv[]) { int depth = 1; if (argc >= 1) { @@ -595,24 +680,36 @@ int NinjaMain::ToolRules(const Options* options, int argc, char* argv[]) { // Print rules - typedef map Rules; + typedef map> Rules; const Rules& rules = state_.bindings_.GetRules(); for (Rules::const_iterator i = rules.begin(); i != rules.end(); ++i) { printf("%s", i->first.c_str()); if (print_description) { - const Rule* rule = i->second; + const Rule* rule = i->second.get(); const EvalString* description = rule->GetBinding("description"); if (description != NULL) { printf(": %s", description->Unparse().c_str()); } } printf("\n"); + fflush(stdout); } return 0; } +#ifdef _WIN32 +int NinjaMain::ToolWinCodePage(const Options* options, int argc, char* argv[]) { + if (argc != 0) { + printf("usage: ninja -t wincodepage\n"); + return 1; + } + printf("Build file encoding: %s\n", GetACP() == CP_UTF8? "UTF-8" : "ANSI"); + return 0; +} +#endif + enum PrintCommandMode { PCM_Single, PCM_All }; -void PrintCommands(Edge* edge, set* seen, PrintCommandMode mode) { +void PrintCommands(Edge* edge, EdgeSet* seen, PrintCommandMode mode) { if (!edge) return; if (!seen->insert(edge).second) @@ -629,7 +726,7 @@ void PrintCommands(Edge* edge, set* seen, PrintCommandMode mode) { } int NinjaMain::ToolCommands(const Options* options, int argc, char* argv[]) { - // The clean tool uses getopt, and expects argv[0] to contain the name of + // The commands tool uses getopt, and expects argv[0] to contain the name of // the tool, i.e. "commands". ++argc; --argv; @@ -663,13 +760,160 @@ int NinjaMain::ToolCommands(const Options* options, int argc, char* argv[]) { return 1; } - set seen; + EdgeSet seen; for (vector::iterator in = nodes.begin(); in != nodes.end(); ++in) PrintCommands((*in)->in_edge(), &seen, mode); return 0; } +int NinjaMain::ToolInputs(const Options* options, int argc, char* argv[]) { + // The inputs tool uses getopt, and expects argv[0] to contain the name of + // the tool, i.e. "inputs". + argc++; + argv--; + + bool print0 = false; + bool shell_escape = true; + bool dependency_order = false; + + optind = 1; + int opt; + const option kLongOptions[] = { { "help", no_argument, NULL, 'h' }, + { "no-shell-escape", no_argument, NULL, 'E' }, + { "print0", no_argument, NULL, '0' }, + { "dependency-order", no_argument, NULL, + 'd' }, + { NULL, 0, NULL, 0 } }; + while ((opt = getopt_long(argc, argv, "h0Ed", kLongOptions, NULL)) != -1) { + switch (opt) { + case 'd': + dependency_order = true; + break; + case 'E': + shell_escape = false; + break; + case '0': + print0 = true; + break; + case 'h': + default: + // clang-format off + printf( +"Usage '-t inputs [options] [targets]\n" +"\n" +"List all inputs used for a set of targets, sorted in dependency order.\n" +"Note that by default, results are shell escaped, and sorted alphabetically,\n" +"and never include validation target paths.\n\n" +"Options:\n" +" -h, --help Print this message.\n" +" -0, --print0 Use \\0, instead of \\n as a line terminator.\n" +" -E, --no-shell-escape Do not shell escape the result.\n" +" -d, --dependency-order Sort results by dependency order.\n" + ); + // clang-format on + return 1; + } + } + argv += optind; + argc -= optind; + + std::vector nodes; + std::string err; + if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) { + Error("%s", err.c_str()); + return 1; + } + + InputsCollector collector; + for (const Node* node : nodes) + collector.VisitNode(node); + + std::vector inputs = collector.GetInputsAsStrings(shell_escape); + if (!dependency_order) + std::sort(inputs.begin(), inputs.end()); + + if (print0) { + for (const std::string& input : inputs) { + fwrite(input.c_str(), input.size(), 1, stdout); + fputc('\0', stdout); + } + fflush(stdout); + } else { + for (const std::string& input : inputs) + puts(input.c_str()); + } + return 0; +} + +int NinjaMain::ToolMultiInputs(const Options* options, int argc, char* argv[]) { + // The inputs tool uses getopt, and expects argv[0] to contain the name of + // the tool, i.e. "inputs". + argc++; + argv--; + + optind = 1; + int opt; + char terminator = '\n'; + const char* delimiter = "\t"; + const option kLongOptions[] = { { "help", no_argument, NULL, 'h' }, + { "delimiter", required_argument, NULL, + 'd' }, + { "print0", no_argument, NULL, '0' }, + { NULL, 0, NULL, 0 } }; + while ((opt = getopt_long(argc, argv, "d:h0", kLongOptions, NULL)) != -1) { + switch (opt) { + case 'd': + delimiter = optarg; + break; + case '0': + terminator = '\0'; + break; + case 'h': + default: + // clang-format off + printf( +"Usage '-t multi-inputs [options] [targets]\n" +"\n" +"Print one or more sets of inputs required to build targets, sorted in dependency order.\n" +"The tool works like inputs tool but with addition of the target for each line.\n" +"The output will be a series of lines with the following elements:\n" +" \n" +"Note that a given input may appear for several targets if it is used by more than one targets.\n" +"Options:\n" +" -h, --help Print this message.\n" +" -d --delimiter=DELIM Use DELIM instead of TAB for field delimiter.\n" +" -0, --print0 Use \\0, instead of \\n as a line terminator.\n" + ); + // clang-format on + return 1; + } + } + argv += optind; + argc -= optind; + + std::vector nodes; + std::string err; + if (!CollectTargetsFromArgs(argc, argv, &nodes, &err)) { + Error("%s", err.c_str()); + return 1; + } + + for (const Node* node : nodes) { + InputsCollector collector; + + collector.VisitNode(node); + std::vector inputs = collector.GetInputsAsStrings(); + + for (const std::string& input : inputs) { + printf("%s%s%s", node->path().c_str(), delimiter, input.c_str()); + fputc(terminator, stdout); + } + } + + return 0; +} + int NinjaMain::ToolClean(const Options* options, int argc, char* argv[]) { // The clean tool uses getopt, and expects argv[0] to contain the name of // the tool, i.e. "clean". @@ -719,20 +963,17 @@ int NinjaMain::ToolClean(const Options* options, int argc, char* argv[]) { } } -void EncodeJSONString(const char *str) { - while (*str) { - if (*str == '"' || *str == '\\') - putchar('\\'); - putchar(*str); - str++; - } +int NinjaMain::ToolCleanDead(const Options* options, int argc, char* argv[]) { + Cleaner cleaner(&state_, config_, &disk_interface_); + return cleaner.CleanDead(build_log_.entries()); } enum EvaluateCommandMode { ECM_NORMAL, ECM_EXPAND_RSPFILE }; -string EvaluateCommandWithRspfile(Edge* edge, EvaluateCommandMode mode) { +std::string EvaluateCommandWithRspfile(const Edge* edge, + const EvaluateCommandMode mode) { string command = edge->EvaluateCommand(); if (mode == ECM_NORMAL) return command; @@ -742,7 +983,10 @@ string EvaluateCommandWithRspfile(Edge* edge, EvaluateCommandMode mode) { return command; size_t index = command.find(rspfile); - if (index == 0 || index == string::npos || command[index - 1] != '@') + if (index == 0 || index == string::npos || + (command[index - 1] != '@' && + command.find("--option-file=") != index - 14 && + command.find("-f ") != index - 3)) return command; string rspfile_content = edge->GetBinding("rspfile_content"); @@ -752,10 +996,29 @@ string EvaluateCommandWithRspfile(Edge* edge, EvaluateCommandMode mode) { rspfile_content.replace(newline_index, 1, 1, ' '); ++newline_index; } - command.replace(index - 1, rspfile.length() + 1, rspfile_content); + if (command[index - 1] == '@') { + command.replace(index - 1, rspfile.length() + 1, rspfile_content); + } else if (command.find("-f ") == index - 3) { + command.replace(index - 3, rspfile.length() + 3, rspfile_content); + } else { // --option-file syntax + command.replace(index - 14, rspfile.length() + 14, rspfile_content); + } return command; } +void PrintOneCompdbObject(std::string const& directory, const Edge* const edge, + const EvaluateCommandMode eval_mode) { + printf("\n {\n \"directory\": \""); + PrintJSONString(directory); + printf("\",\n \"command\": \""); + PrintJSONString(EvaluateCommandWithRspfile(edge, eval_mode)); + printf("\",\n \"file\": \""); + PrintJSONString(edge->inputs_[0]->path()); + printf("\",\n \"output\": \""); + PrintJSONString(edge->outputs_[0]->path()); + printf("\"\n }"); +} + int NinjaMain::ToolCompilationDatabase(const Options* options, int argc, char* argv[]) { // The compdb tool uses getopt, and expects argv[0] to contain the name of @@ -788,38 +1051,27 @@ int NinjaMain::ToolCompilationDatabase(const Options* options, int argc, argc -= optind; bool first = true; - vector cwd; - - do { - cwd.resize(cwd.size() + 1024); - errno = 0; - } while (!getcwd(&cwd[0], cwd.size()) && errno == ERANGE); - if (errno != 0 && errno != ERANGE) { - Error("cannot determine working directory: %s", strerror(errno)); - return 1; - } + std::string directory = GetWorkingDirectory(); putchar('['); - for (vector::iterator e = state_.edges_.begin(); - e != state_.edges_.end(); ++e) { - if ((*e)->inputs_.empty()) + for (const Edge* edge : state_.edges_) { + if (edge->inputs_.empty()) continue; - for (int i = 0; i != argc; ++i) { - if ((*e)->rule_->name() == argv[i]) { - if (!first) - putchar(','); - - printf("\n {\n \"directory\": \""); - EncodeJSONString(&cwd[0]); - printf("\",\n \"command\": \""); - EncodeJSONString(EvaluateCommandWithRspfile(*e, eval_mode).c_str()); - printf("\",\n \"file\": \""); - EncodeJSONString((*e)->inputs_[0]->path().c_str()); - printf("\",\n \"output\": \""); - EncodeJSONString((*e)->outputs_[0]->path().c_str()); - printf("\"\n }"); - - first = false; + if (argc == 0) { + if (!first) { + putchar(','); + } + PrintOneCompdbObject(directory, edge, eval_mode); + first = false; + } else { + for (int i = 0; i != argc; ++i) { + if (edge->rule_->name() == argv[i]) { + if (!first) { + putchar(','); + } + PrintOneCompdbObject(directory, edge, eval_mode); + first = false; + } } } } @@ -839,6 +1091,176 @@ int NinjaMain::ToolRecompact(const Options* options, int argc, char* argv[]) { return 0; } +int NinjaMain::ToolRestat(const Options* options, int argc, char* argv[]) { + // The restat tool uses getopt, and expects argv[0] to contain the name of the + // tool, i.e. "restat" + argc++; + argv--; + + optind = 1; + int opt; + while ((opt = getopt(argc, argv, const_cast("h"))) != -1) { + switch (opt) { + case 'h': + default: + printf("usage: ninja -t restat [outputs]\n"); + return 1; + } + } + argv += optind; + argc -= optind; + + if (!EnsureBuildDirExists()) + return 1; + + string log_path = ".ninja_log"; + if (!build_dir_.empty()) + log_path = build_dir_ + "/" + log_path; + + string err; + const LoadStatus status = build_log_.Load(log_path, &err); + if (status == LOAD_ERROR) { + Error("loading build log %s: %s", log_path.c_str(), err.c_str()); + return EXIT_FAILURE; + } + if (status == LOAD_NOT_FOUND) { + // Nothing to restat, ignore this + return EXIT_SUCCESS; + } + if (!err.empty()) { + // Hack: Load() can return a warning via err by returning LOAD_SUCCESS. + Warning("%s", err.c_str()); + err.clear(); + } + + bool success = build_log_.Restat(log_path, disk_interface_, argc, argv, &err); + if (!success) { + Error("failed recompaction: %s", err.c_str()); + return EXIT_FAILURE; + } + + if (!config_.dry_run) { + if (!build_log_.OpenForWrite(log_path, *this, &err)) { + Error("opening build log: %s", err.c_str()); + return EXIT_FAILURE; + } + } + + return EXIT_SUCCESS; +} + +struct CompdbTargets { + enum class Action { kDisplayHelpAndExit, kEmitCommands }; + + Action action; + EvaluateCommandMode eval_mode = ECM_NORMAL; + + std::vector targets; + + static CompdbTargets CreateFromArgs(int argc, char* argv[]) { + // + // grammar: + // ninja -t compdb-targets [-hx] target [targets] + // + CompdbTargets ret; + + // getopt_long() expects argv[0] to contain the name of + // the tool, i.e. "compdb-targets". + argc++; + argv--; + + // Phase 1: parse options: + optind = 1; // see `man 3 getopt` for documentation on optind + int opt; + while ((opt = getopt(argc, argv, const_cast("hx"))) != -1) { + switch (opt) { + case 'x': + ret.eval_mode = ECM_EXPAND_RSPFILE; + break; + case 'h': + default: + ret.action = CompdbTargets::Action::kDisplayHelpAndExit; + return ret; + } + } + + // Phase 2: parse operands: + int const targets_begin = optind; + int const targets_end = argc; + + if (targets_begin == targets_end) { + Error("compdb-targets expects the name of at least one target"); + ret.action = CompdbTargets::Action::kDisplayHelpAndExit; + } else { + ret.action = CompdbTargets::Action::kEmitCommands; + for (int i = targets_begin; i < targets_end; ++i) { + ret.targets.push_back(argv[i]); + } + } + + return ret; + } +}; + +void PrintCompdb(std::string const& directory, std::vector const& edges, + const EvaluateCommandMode eval_mode) { + putchar('['); + + bool first = true; + for (const Edge* edge : edges) { + if (edge->is_phony() || edge->inputs_.empty()) + continue; + if (!first) + putchar(','); + PrintOneCompdbObject(directory, edge, eval_mode); + first = false; + } + + puts("\n]"); +} + +int NinjaMain::ToolCompilationDatabaseForTargets(const Options* options, + int argc, char* argv[]) { + auto compdb = CompdbTargets::CreateFromArgs(argc, argv); + + switch (compdb.action) { + case CompdbTargets::Action::kDisplayHelpAndExit: { + printf( + "usage: ninja -t compdb [-hx] target [targets]\n" + "\n" + "options:\n" + " -h display this help message\n" + " -x expand @rspfile style response file invocations\n"); + return 1; + } + + case CompdbTargets::Action::kEmitCommands: { + CommandCollector collector; + + for (const std::string& target_arg : compdb.targets) { + std::string err; + Node* node = CollectTarget(target_arg.c_str(), &err); + if (!node) { + Fatal("%s", err.c_str()); + return 1; + } + if (!node->in_edge()) { + Fatal( + "'%s' is not a target " + "(i.e. it is not an output of any `build` statement)", + node->path().c_str()); + } + collector.CollectFrom(node); + } + + std::string directory = GetWorkingDirectory(); + PrintCompdb(directory, collector.in_edges, compdb.eval_mode); + } break; + } + + return 0; +} + int NinjaMain::ToolUrtle(const Options* options, int argc, char** argv) { // RLE encoded. const char* urtle = @@ -871,16 +1293,22 @@ const Tool* ChooseTool(const string& tool_name) { static const Tool kTools[] = { { "browse", "browse dependency graph in a web browser", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolBrowse }, -#if defined(_MSC_VER) - { "msvc", "build helper for MSVC cl.exe (EXPERIMENTAL)", +#ifdef _WIN32 + { "msvc", "build helper for MSVC cl.exe (DEPRECATED)", Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolMSVC }, #endif { "clean", "clean built files", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolClean }, { "commands", "list all commands required to rebuild given targets", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCommands }, + { "inputs", "list all inputs required to rebuild given targets", + Tool::RUN_AFTER_LOAD, &NinjaMain::ToolInputs}, + { "multi-inputs", "print one or more sets of inputs required to build targets", + Tool::RUN_AFTER_LOAD, &NinjaMain::ToolMultiInputs}, { "deps", "show dependencies stored in the deps log", Tool::RUN_AFTER_LOGS, &NinjaMain::ToolDeps }, + { "missingdeps", "check deps log dependencies on generated files", + Tool::RUN_AFTER_LOGS, &NinjaMain::ToolMissingDeps }, { "graph", "output graphviz dot file for targets", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolGraph }, { "query", "show inputs/outputs for a path", @@ -889,12 +1317,23 @@ const Tool* ChooseTool(const string& tool_name) { Tool::RUN_AFTER_LOAD, &NinjaMain::ToolTargets }, { "compdb", "dump JSON compilation database to stdout", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCompilationDatabase }, + { "compdb-targets", + "dump JSON compilation database for a given list of targets to stdout", + Tool::RUN_AFTER_LOAD, &NinjaMain::ToolCompilationDatabaseForTargets }, { "recompact", "recompacts ninja-internal data structures", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRecompact }, + { "restat", "restats all outputs in the build log", + Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolRestat }, { "rules", "list all rules", Tool::RUN_AFTER_LOAD, &NinjaMain::ToolRules }, + { "cleandead", "clean built files that are no longer produced by the manifest", + Tool::RUN_AFTER_LOGS, &NinjaMain::ToolCleanDead }, { "urtle", NULL, Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolUrtle }, +#ifdef _WIN32 + { "wincodepage", "print the Windows code page used by ninja", + Tool::RUN_AFTER_FLAGS, &NinjaMain::ToolWinCodePage }, +#endif { NULL, NULL, Tool::RUN_AFTER_FLAGS, NULL } }; @@ -902,7 +1341,7 @@ const Tool* ChooseTool(const string& tool_name) { printf("ninja subtools:\n"); for (const Tool* tool = &kTools[0]; tool->name; ++tool) { if (tool->desc) - printf("%10s %s\n", tool->name, tool->desc); + printf("%11s %s\n", tool->name, tool->desc); } return NULL; } @@ -969,38 +1408,31 @@ bool DebugEnable(const string& name) { } } -/// Set a warning flag. Returns false if Ninja should exit instead of +/// Set a warning flag. Returns false if Ninja should exit instead of /// continuing. bool WarningEnable(const string& name, Options* options) { if (name == "list") { printf("warning flags:\n" -" dupbuild={err,warn} multiple build lines for one target\n" " phonycycle={err,warn} phony build statement references itself\n" -" depfilemulti={err,warn} depfile has multiple output paths on separate lines\n" ); return false; - } else if (name == "dupbuild=err") { - options->dupe_edges_should_err = true; - return true; - } else if (name == "dupbuild=warn") { - options->dupe_edges_should_err = false; - return true; } else if (name == "phonycycle=err") { options->phony_cycle_should_err = true; return true; } else if (name == "phonycycle=warn") { options->phony_cycle_should_err = false; return true; - } else if (name == "depfilemulti=err") { - options->depfile_distinct_target_lines_should_err = true; + } else if (name == "dupbuild=err" || + name == "dupbuild=warn") { + Warning("deprecated warning 'dupbuild'"); return true; - } else if (name == "depfilemulti=warn") { - options->depfile_distinct_target_lines_should_err = false; + } else if (name == "depfilemulti=err" || + name == "depfilemulti=warn") { + Warning("deprecated warning 'depfilemulti'"); return true; } else { - const char* suggestion = - SpellcheckString(name.c_str(), "dupbuild=err", "dupbuild=warn", - "phonycycle=err", "phonycycle=warn", NULL); + const char* suggestion = SpellcheckString(name.c_str(), "phonycycle=err", + "phonycycle=warn", nullptr); if (suggestion) { Error("unknown warning flag '%s', did you mean '%s'?", name.c_str(), suggestion); @@ -1017,17 +1449,21 @@ bool NinjaMain::OpenBuildLog(bool recompact_only) { log_path = build_dir_ + "/" + log_path; string err; - if (!build_log_.Load(log_path, &err)) { + const LoadStatus status = build_log_.Load(log_path, &err); + if (status == LOAD_ERROR) { Error("loading build log %s: %s", log_path.c_str(), err.c_str()); return false; } if (!err.empty()) { - // Hack: Load() can return a warning via err by returning true. + // Hack: Load() can return a warning via err by returning LOAD_SUCCESS. Warning("%s", err.c_str()); err.clear(); } if (recompact_only) { + if (status == LOAD_NOT_FOUND) { + return true; + } bool success = build_log_.Recompact(log_path, *this, &err); if (!success) Error("failed recompaction: %s", err.c_str()); @@ -1052,17 +1488,21 @@ bool NinjaMain::OpenDepsLog(bool recompact_only) { path = build_dir_ + "/" + path; string err; - if (!deps_log_.Load(path, &state_, &err)) { + const LoadStatus status = deps_log_.Load(path, &state_, &err); + if (status == LOAD_ERROR) { Error("loading deps log %s: %s", path.c_str(), err.c_str()); return false; } if (!err.empty()) { - // Hack: Load() can return a warning via err by returning true. + // Hack: Load() can return a warning via err by returning LOAD_SUCCESS. Warning("%s", err.c_str()); err.clear(); } if (recompact_only) { + if (status == LOAD_NOT_FOUND) { + return true; + } bool success = deps_log_.Recompact(path, &err); if (!success) Error("failed recompaction: %s", err.c_str()); @@ -1101,22 +1541,23 @@ bool NinjaMain::EnsureBuildDirExists() { return true; } -int NinjaMain::RunBuild(int argc, char** argv) { +ExitStatus NinjaMain::RunBuild(int argc, char** argv, Status* status) { string err; vector targets; if (!CollectTargetsFromArgs(argc, argv, &targets, &err)) { - Error("%s", err.c_str()); - return 1; + status->Error("%s", err.c_str()); + return ExitFailure; } disk_interface_.AllowStatCache(g_experimental_statcache); - Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_); + Builder builder(&state_, config_, &build_log_, &deps_log_, &disk_interface_, + status, start_time_millis_); for (size_t i = 0; i < targets.size(); ++i) { if (!builder.AddTarget(targets[i], &err)) { if (!err.empty()) { - Error("%s", err.c_str()); - return 1; + status->Error("%s", err.c_str()); + return ExitFailure; } else { // Added a target that is already up-to-date; not really // an error. @@ -1128,19 +1569,21 @@ int NinjaMain::RunBuild(int argc, char** argv) { disk_interface_.AllowStatCache(false); if (builder.AlreadyUpToDate()) { - printf("ninja: no work to do.\n"); - return 0; + if (config_.verbosity != BuildConfig::NO_STATUS_UPDATE) { + status->Info("no work to do."); + } + return ExitSuccess; } - if (!builder.Build(&err)) { - printf("ninja: build stopped: %s.\n", err.c_str()); + ExitStatus exit_status = builder.Build(&err); + if (exit_status != ExitSuccess) { + status->Info("build stopped: %s.", err.c_str()); if (err.find("interrupted by user") != string::npos) { - return 2; + return ExitInterrupted; } - return 1; } - return 0; + return exit_status; } #ifdef _MSC_VER @@ -1166,17 +1609,35 @@ int ExceptionFilter(unsigned int code, struct _EXCEPTION_POINTERS *ep) { #endif // _MSC_VER +class DeferGuessParallelism { + public: + bool needGuess; + BuildConfig* config; + + DeferGuessParallelism(BuildConfig* config) + : needGuess(true), config(config) {} + + void Refresh() { + if (needGuess) { + needGuess = false; + config->parallelism = GuessParallelism(); + } + } + ~DeferGuessParallelism() { Refresh(); } +}; + /// Parse argv for command-line options. /// Returns an exit code, or -1 if Ninja should continue. int ReadFlags(int* argc, char*** argv, Options* options, BuildConfig* config) { - config->parallelism = GuessParallelism(); + DeferGuessParallelism deferGuessParallelism(config); - enum { OPT_VERSION = 1 }; + enum { OPT_VERSION = 1, OPT_QUIET = 2 }; const option kLongOptions[] = { { "help", no_argument, NULL, 'h' }, { "version", no_argument, NULL, OPT_VERSION }, { "verbose", no_argument, NULL, 'v' }, + { "quiet", no_argument, NULL, OPT_QUIET }, { NULL, 0, NULL, 0 } }; @@ -1201,6 +1662,7 @@ int ReadFlags(int* argc, char*** argv, // We want to run N jobs in parallel. For N = 0, INT_MAX // is close enough to infinite for most sane builds. config->parallelism = value > 0 ? value : INT_MAX; + deferGuessParallelism.needGuess = false; break; } case 'k': { @@ -1234,6 +1696,9 @@ int ReadFlags(int* argc, char*** argv, case 'v': config->verbosity = BuildConfig::VERBOSE; break; + case OPT_QUIET: + config->verbosity = BuildConfig::NO_STATUS_UPDATE; + break; case 'w': if (!WarningEnable(optarg, options)) return 1; @@ -1246,6 +1711,7 @@ int ReadFlags(int* argc, char*** argv, return 0; case 'h': default: + deferGuessParallelism.Refresh(); Usage(*config); return 1; } @@ -1262,7 +1728,6 @@ NORETURN void real_main(int argc, char** argv) { BuildConfig config; Options options = {}; options.input_file = "build.ninja"; - options.dupe_edges_should_err = true; setvbuf(stdout, NULL, _IOLBF, BUFSIZ); const char* ninja_command = argv[0]; @@ -1271,10 +1736,7 @@ NORETURN void real_main(int argc, char** argv) { if (exit_code >= 0) exit(exit_code); - if (options.depfile_distinct_target_lines_should_err) { - config.depfile_parser_options.depfile_distinct_target_lines_action_ = - kDepfileDistinctTargetLinesActionError; - } + Status* status = Status::factory(config); if (options.working_dir) { // The formatting of this string, complete with funny quotes, is @@ -1282,8 +1744,8 @@ NORETURN void real_main(int argc, char** argv) { // subsequent commands. // Don't print this if a tool is being used, so that tool output // can be piped into a file without this string showing up. - if (!options.tool) - printf("ninja: Entering directory `%s'\n", options.working_dir); + if (!options.tool && config.verbosity != BuildConfig::NO_STATUS_UPDATE) + status->Info("Entering directory `%s'", options.working_dir); if (chdir(options.working_dir) < 0) { Fatal("chdir to '%s' - %s", options.working_dir, strerror(errno)); } @@ -1302,16 +1764,13 @@ NORETURN void real_main(int argc, char** argv) { NinjaMain ninja(ninja_command, config); ManifestParserOptions parser_opts; - if (options.dupe_edges_should_err) { - parser_opts.dupe_edge_action_ = kDupeEdgeActionError; - } if (options.phony_cycle_should_err) { parser_opts.phony_cycle_action_ = kPhonyCycleActionError; } ManifestParser parser(&ninja.state_, &ninja.disk_interface_, parser_opts); string err; if (!parser.Load(options.input_file, &err)) { - Error("%s", err.c_str()); + status->Error("%s", err.c_str()); exit(1); } @@ -1328,7 +1787,7 @@ NORETURN void real_main(int argc, char** argv) { exit((ninja.*options.tool->func)(&options, argc, argv)); // Attempt to rebuild the manifest before building anything else - if (ninja.RebuildManifest(options.input_file, &err)) { + if (ninja.RebuildManifest(options.input_file, &err, status)) { // In dry_run mode the regeneration will succeed without changing the // manifest forever. Better to return immediately. if (config.dry_run) @@ -1336,17 +1795,19 @@ NORETURN void real_main(int argc, char** argv) { // Start the build over with the new manifest. continue; } else if (!err.empty()) { - Error("rebuilding '%s': %s", options.input_file, err.c_str()); + status->Error("rebuilding '%s': %s", options.input_file, err.c_str()); exit(1); } - int result = ninja.RunBuild(argc, argv); + ninja.ParsePreviousElapsedTimes(); + + ExitStatus result = ninja.RunBuild(argc, argv, status); if (g_metrics) ninja.DumpMetrics(); exit(result); } - Error("manifest '%s' still dirty after %d tries\n", + status->Error("manifest '%s' still dirty after %d tries, perhaps system time is not set", options.input_file, kCycleLimit); exit(1); } diff --git a/src/ninja_test.cc b/src/ninja_test.cc index d642c5c90d..7616c85bcd 100644 --- a/src/ninja_test.cc +++ b/src/ninja_test.cc @@ -12,149 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include -#include -#include - -#ifdef _WIN32 -#include "getopt.h" -#elif defined(_AIX) -#include "getopt.h" -#include -#else -#include -#endif - -#include "test.h" -#include "line_printer.h" - -struct RegisteredTest { - testing::Test* (*factory)(); - const char *name; - bool should_run; -}; -// This can't be a vector because tests call RegisterTest from static -// initializers and the order static initializers run it isn't specified. So -// the vector constructor isn't guaranteed to run before all of the -// RegisterTest() calls. -static RegisteredTest tests[10000]; -testing::Test* g_current_test; -static int ntests; -static LinePrinter printer; - -void RegisterTest(testing::Test* (*factory)(), const char* name) { - tests[ntests].factory = factory; - tests[ntests++].name = name; -} - -namespace { -string StringPrintf(const char* format, ...) { - const int N = 1024; - char buf[N]; - - va_list ap; - va_start(ap, format); - vsnprintf(buf, N, format, ap); - va_end(ap); - - return buf; -} - -void Usage() { - fprintf(stderr, -"usage: ninja_tests [options]\n" -"\n" -"options:\n" -" --gtest_filter=POSTIVE_PATTERN[-NEGATIVE_PATTERN]\n" -" Run tests whose names match the positive but not the negative pattern.\n" -" '*' matches any substring. (gtest's ':', '?' are not implemented).\n"); -} - -bool PatternMatchesString(const char* pattern, const char* str) { - switch (*pattern) { - case '\0': - case '-': return *str == '\0'; - case '*': return (*str != '\0' && PatternMatchesString(pattern, str + 1)) || - PatternMatchesString(pattern + 1, str); - default: return *pattern == *str && - PatternMatchesString(pattern + 1, str + 1); - } -} - -bool TestMatchesFilter(const char* test, const char* filter) { - // Split --gtest_filter at '-' into positive and negative filters. - const char* const dash = strchr(filter, '-'); - const char* pos = dash == filter ? "*" : filter; //Treat '-test1' as '*-test1' - const char* neg = dash ? dash + 1 : ""; - return PatternMatchesString(pos, test) && !PatternMatchesString(neg, test); -} - -bool ReadFlags(int* argc, char*** argv, const char** test_filter) { - enum { OPT_GTEST_FILTER = 1 }; - const option kLongOptions[] = { - { "gtest_filter", required_argument, NULL, OPT_GTEST_FILTER }, - { NULL, 0, NULL, 0 } - }; - - int opt; - while ((opt = getopt_long(*argc, *argv, "h", kLongOptions, NULL)) != -1) { - switch (opt) { - case OPT_GTEST_FILTER: - if (strchr(optarg, '?') == NULL && strchr(optarg, ':') == NULL) { - *test_filter = optarg; - break; - } // else fall through. - default: - Usage(); - return false; - } - } - *argv += optind; - *argc -= optind; - return true; -} - -} // namespace - -bool testing::Test::Check(bool condition, const char* file, int line, - const char* error) { - if (!condition) { - printer.PrintOnNewLine( - StringPrintf("*** Failure in %s:%d\n%s\n", file, line, error)); - failed_ = true; - } - return condition; -} +#include int main(int argc, char **argv) { - int tests_started = 0; - - const char* test_filter = "*"; - if (!ReadFlags(&argc, &argv, &test_filter)) - return 1; - - int nactivetests = 0; - for (int i = 0; i < ntests; i++) - if ((tests[i].should_run = TestMatchesFilter(tests[i].name, test_filter))) - ++nactivetests; - - bool passed = true; - for (int i = 0; i < ntests; i++) { - if (!tests[i].should_run) continue; - - ++tests_started; - testing::Test* test = tests[i].factory(); - printer.Print( - StringPrintf("[%d/%d] %s", tests_started, nactivetests, tests[i].name), - LinePrinter::ELIDE); - test->SetUp(); - test->Run(); - test->TearDown(); - if (test->Failed()) - passed = false; - delete test; - } - - printer.PrintOnNewLine(passed ? "passed\n" : "failed\n"); - return passed ? EXIT_SUCCESS : EXIT_FAILURE; + testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); } diff --git a/src/parser.cc b/src/parser.cc index 745c532d72..139a347ac3 100644 --- a/src/parser.cc +++ b/src/parser.cc @@ -17,8 +17,13 @@ #include "disk_interface.h" #include "metrics.h" +using namespace std; + bool Parser::Load(const string& filename, string* err, Lexer* parent) { - METRIC_RECORD(".ninja parse"); + // If |parent| is not NULL, metrics collection has been started by a parent + // Parser::Load() in our call stack. Do not start a new one here to avoid + // over-counting parsing times. + METRIC_RECORD_IF(".ninja parse", parent == NULL); string contents; string read_err; if (file_reader_->ReadFile(filename, &contents, &read_err) != @@ -29,13 +34,6 @@ bool Parser::Load(const string& filename, string* err, Lexer* parent) { return false; } - // The lexer needs a nul byte at the end of its input, to know when it's done. - // It takes a StringPiece, and StringPiece's string constructor uses - // string::data(). data()'s return value isn't guaranteed to be - // null-terminated (although in practice - libc++, libstdc++, msvc's stl -- - // it is, and C++11 demands that too), so add an explicit nul byte. - contents.resize(contents.size() + 1); - return Parse(filename, contents, err); } diff --git a/src/parser.h b/src/parser.h index e2d2b97fd4..75990aae7d 100644 --- a/src/parser.h +++ b/src/parser.h @@ -17,8 +17,6 @@ #include -using namespace std; - #include "lexer.h" struct FileReader; @@ -28,14 +26,15 @@ struct State; struct Parser { Parser(State* state, FileReader* file_reader) : state_(state), file_reader_(file_reader) {} + virtual ~Parser() {} /// Load and parse a file. - bool Load(const string& filename, string* err, Lexer* parent = NULL); + bool Load(const std::string& filename, std::string* err, Lexer* parent = NULL); protected: /// If the next token is not \a expected, produce an error string /// saying "expected foo, got bar". - bool ExpectToken(Lexer::Token expected, string* err); + bool ExpectToken(Lexer::Token expected, std::string* err); State* state_; FileReader* file_reader_; @@ -43,8 +42,8 @@ struct Parser { private: /// Parse a file, given its contents as a string. - virtual bool Parse(const string& filename, const string& input, - string* err) = 0; + virtual bool Parse(const std::string& filename, const std::string& input, + std::string* err) = 0; }; #endif // NINJA_PARSER_H_ diff --git a/src/real_command_runner.cc b/src/real_command_runner.cc new file mode 100644 index 0000000000..453652f5e5 --- /dev/null +++ b/src/real_command_runner.cc @@ -0,0 +1,98 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "build.h" +#include "subprocess.h" + +struct RealCommandRunner : public CommandRunner { + explicit RealCommandRunner(const BuildConfig& config) : config_(config) {} + size_t CanRunMore() const override; + bool StartCommand(Edge* edge) override; + bool WaitForCommand(Result* result) override; + std::vector GetActiveEdges() override; + void Abort() override; + + const BuildConfig& config_; + SubprocessSet subprocs_; + std::map subproc_to_edge_; +}; + +std::vector RealCommandRunner::GetActiveEdges() { + std::vector edges; + for (std::map::iterator e = + subproc_to_edge_.begin(); + e != subproc_to_edge_.end(); ++e) + edges.push_back(e->second); + return edges; +} + +void RealCommandRunner::Abort() { + subprocs_.Clear(); +} + +size_t RealCommandRunner::CanRunMore() const { + size_t subproc_number = + subprocs_.running_.size() + subprocs_.finished_.size(); + + int64_t capacity = config_.parallelism - subproc_number; + + if (config_.max_load_average > 0.0f) { + int load_capacity = config_.max_load_average - GetLoadAverage(); + if (load_capacity < capacity) + capacity = load_capacity; + } + + if (capacity < 0) + capacity = 0; + + if (capacity == 0 && subprocs_.running_.empty()) + // Ensure that we make progress. + capacity = 1; + + return capacity; +} + +bool RealCommandRunner::StartCommand(Edge* edge) { + std::string command = edge->EvaluateCommand(); + Subprocess* subproc = subprocs_.Add(command, edge->use_console()); + if (!subproc) + return false; + subproc_to_edge_.insert(std::make_pair(subproc, edge)); + + return true; +} + +bool RealCommandRunner::WaitForCommand(Result* result) { + Subprocess* subproc; + while ((subproc = subprocs_.NextFinished()) == NULL) { + bool interrupted = subprocs_.DoWork(); + if (interrupted) + return false; + } + + result->status = subproc->Finish(); + result->output = subproc->GetOutput(); + + std::map::iterator e = + subproc_to_edge_.find(subproc); + result->edge = e->second; + subproc_to_edge_.erase(e); + + delete subproc; + return true; +} + +CommandRunner* CommandRunner::factory(const BuildConfig& config) { + return new RealCommandRunner(config); +} diff --git a/src/state.cc b/src/state.cc index 74cf4c1c94..a8a1482d39 100644 --- a/src/state.cc +++ b/src/state.cc @@ -19,9 +19,9 @@ #include "edit_distance.h" #include "graph.h" -#include "metrics.h" #include "util.h" +using namespace std; void Pool::EdgeScheduled(const Edge& edge) { if (depth_ != 0) @@ -38,13 +38,13 @@ void Pool::DelayEdge(Edge* edge) { delayed_.insert(edge); } -void Pool::RetrieveReadyEdges(set* ready_queue) { +void Pool::RetrieveReadyEdges(EdgePriorityQueue* ready_queue) { DelayedEdges::iterator it = delayed_.begin(); while (it != delayed_.end()) { Edge* edge = *it; if (current_use_ + edge->weight() > depth_) break; - ready_queue->insert(edge); + ready_queue->push(edge); EdgeScheduled(*edge); ++it; } @@ -61,20 +61,11 @@ void Pool::Dump() const { } } -// static -bool Pool::WeightedEdgeCmp(const Edge* a, const Edge* b) { - if (!a) return b; - if (!b) return false; - int weight_diff = a->weight() - b->weight(); - return ((weight_diff < 0) || (weight_diff == 0 && a < b)); -} - Pool State::kDefaultPool("", 0); Pool State::kConsolePool("console", 1); -const Rule State::kPhonyRule("phony"); State::State() { - bindings_.AddRule(&kPhonyRule); + bindings_.AddRule(Rule::Phony()); AddPool(&kDefaultPool); AddPool(&kConsolePool); } @@ -96,6 +87,7 @@ Edge* State::AddEdge(const Rule* rule) { edge->rule_ = rule; edge->pool_ = &State::kDefaultPool; edge->env_ = &bindings_; + edge->id_ = edges_.size(); edges_.push_back(edge); return edge; } @@ -110,7 +102,6 @@ Node* State::GetNode(StringPiece path, uint64_t slash_bits) { } Node* State::LookupNode(StringPiece path) const { - METRIC_RECORD("lookup node"); Paths::const_iterator i = paths_.find(path); if (i != paths_.end()) return i->second; @@ -136,19 +127,35 @@ Node* State::SpellcheckNode(const string& path) { void State::AddIn(Edge* edge, StringPiece path, uint64_t slash_bits) { Node* node = GetNode(path, slash_bits); + node->set_generated_by_dep_loader(false); edge->inputs_.push_back(node); node->AddOutEdge(edge); } -bool State::AddOut(Edge* edge, StringPiece path, uint64_t slash_bits) { +bool State::AddOut(Edge* edge, StringPiece path, uint64_t slash_bits, + std::string* err) { Node* node = GetNode(path, slash_bits); - if (node->in_edge()) + if (Edge* other = node->in_edge()) { + if (other == edge) { + *err = path.AsString() + " is defined as an output multiple times"; + } else { + *err = "multiple rules generate " + path.AsString(); + } return false; + } edge->outputs_.push_back(node); node->set_in_edge(edge); + node->set_generated_by_dep_loader(false); return true; } +void State::AddValidation(Edge* edge, StringPiece path, uint64_t slash_bits) { + Node* node = GetNode(path, slash_bits); + edge->validations_.push_back(node); + node->AddValidationOutEdge(edge); + node->set_generated_by_dep_loader(false); +} + bool State::AddDefault(StringPiece path, string* err) { Node* node = LookupNode(path); if (!node) { diff --git a/src/state.h b/src/state.h index 6fe886c1b2..13e0e818d4 100644 --- a/src/state.h +++ b/src/state.h @@ -19,9 +19,9 @@ #include #include #include -using namespace std; #include "eval_env.h" +#include "graph.h" #include "hash_map.h" #include "util.h" @@ -38,13 +38,13 @@ struct Rule; /// the total scheduled weight diminishes enough (i.e. when a scheduled edge /// completes). struct Pool { - Pool(const string& name, int depth) - : name_(name), current_use_(0), depth_(depth), delayed_(&WeightedEdgeCmp) {} + Pool(const std::string& name, int depth) + : name_(name), current_use_(0), depth_(depth), delayed_() {} // A depth of 0 is infinite bool is_valid() const { return depth_ >= 0; } int depth() const { return depth_; } - const string& name() const { return name_; } + const std::string& name() const { return name_; } int current_use() const { return current_use_; } /// true if the Pool might delay this edge @@ -62,22 +62,32 @@ struct Pool { void DelayEdge(Edge* edge); /// Pool will add zero or more edges to the ready_queue - void RetrieveReadyEdges(set* ready_queue); + void RetrieveReadyEdges(EdgePriorityQueue* ready_queue); /// Dump the Pool and its edges (useful for debugging). void Dump() const; private: - string name_; + std::string name_; /// |current_use_| is the total of the weights of the edges which are /// currently scheduled in the Plan (i.e. the edges in Plan::ready_). int current_use_; int depth_; - static bool WeightedEdgeCmp(const Edge* a, const Edge* b); - - typedef set DelayedEdges; + struct WeightedEdgeCmp { + bool operator()(const Edge* a, const Edge* b) const { + if (!a) return b; + if (!b) return false; + int weight_diff = a->weight() - b->weight(); + if (weight_diff != 0) { + return weight_diff < 0; + } + return EdgePriorityGreater()(a, b); + } + }; + + typedef std::set DelayedEdges; DelayedEdges delayed_; }; @@ -85,22 +95,25 @@ struct Pool { struct State { static Pool kDefaultPool; static Pool kConsolePool; - static const Rule kPhonyRule; State(); void AddPool(Pool* pool); - Pool* LookupPool(const string& pool_name); + Pool* LookupPool(const std::string& pool_name); Edge* AddEdge(const Rule* rule); Node* GetNode(StringPiece path, uint64_t slash_bits); Node* LookupNode(StringPiece path) const; - Node* SpellcheckNode(const string& path); + Node* SpellcheckNode(const std::string& path); + /// Add input / output / validation nodes to a given edge. This also + /// ensures that the generated_by_dep_loader() flag for all these nodes + /// is set to false, to indicate that they come from the input manifest. void AddIn(Edge* edge, StringPiece path, uint64_t slash_bits); - bool AddOut(Edge* edge, StringPiece path, uint64_t slash_bits); - bool AddDefault(StringPiece path, string* error); + bool AddOut(Edge* edge, StringPiece path, uint64_t slash_bits, std::string* err); + void AddValidation(Edge* edge, StringPiece path, uint64_t slash_bits); + bool AddDefault(StringPiece path, std::string* error); /// Reset state. Keeps all nodes and edges, but restores them to the /// state where we haven't yet examined the disk for dirty state. @@ -111,21 +124,21 @@ struct State { /// @return the root node(s) of the graph. (Root nodes have no output edges). /// @param error where to write the error message if somethings went wrong. - vector RootNodes(string* error) const; - vector DefaultNodes(string* error) const; + std::vector RootNodes(std::string* error) const; + std::vector DefaultNodes(std::string* error) const; /// Mapping of path -> Node. typedef ExternalStringHashMap::Type Paths; Paths paths_; /// All the pools used in the graph. - map pools_; + std::map pools_; /// All the edges of the graph. - vector edges_; + std::vector edges_; BindingEnv bindings_; - vector defaults_; + std::vector defaults_; }; #endif // NINJA_STATE_H_ diff --git a/src/state_test.cc b/src/state_test.cc index 458b5196c3..9c1af1c95c 100644 --- a/src/state_test.cc +++ b/src/state_test.cc @@ -16,6 +16,8 @@ #include "state.h" #include "test.h" +using namespace std; + namespace { TEST(State, Basic) { @@ -29,12 +31,12 @@ TEST(State, Basic) { Rule* rule = new Rule("cat"); rule->AddBinding("command", command); - state.bindings_.AddRule(rule); + state.bindings_.AddRule(std::unique_ptr(rule)); Edge* edge = state.AddEdge(rule); state.AddIn(edge, "in1", 0); state.AddIn(edge, "in2", 0); - state.AddOut(edge, "out", 0); + state.AddOut(edge, "out", 0, nullptr); EXPECT_EQ("cat in1 in2 > out", edge->EvaluateCommand()); diff --git a/src/status.h b/src/status.h new file mode 100644 index 0000000000..a873993446 --- /dev/null +++ b/src/status.h @@ -0,0 +1,53 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef NINJA_STATUS_H_ +#define NINJA_STATUS_H_ + +#include +#include "exit_status.h" + +struct BuildConfig; +struct Edge; +struct Explanations; + +/// Abstract interface to object that tracks the status of a build: +/// completion fraction, printing updates. +struct Status { + virtual void EdgeAddedToPlan(const Edge* edge) = 0; + virtual void EdgeRemovedFromPlan(const Edge* edge) = 0; + virtual void BuildEdgeStarted(const Edge* edge, + int64_t start_time_millis) = 0; + virtual void BuildEdgeFinished(Edge* edge, int64_t start_time_millis, + int64_t end_time_millis, ExitStatus exit_code, + const std::string& output) = 0; + virtual void BuildStarted() = 0; + virtual void BuildFinished() = 0; + + /// Set the Explanations instance to use to report explanations, + /// argument can be nullptr if no explanations need to be printed + /// (which is the default). + virtual void SetExplanations(Explanations*) = 0; + + virtual void Info(const char* msg, ...) = 0; + virtual void Warning(const char* msg, ...) = 0; + virtual void Error(const char* msg, ...) = 0; + + virtual ~Status() { } + + /// creates the actual implementation + static Status* factory(const BuildConfig&); +}; + +#endif // NINJA_STATUS_H_ diff --git a/src/status_printer.cc b/src/status_printer.cc new file mode 100644 index 0000000000..e69cd15a40 --- /dev/null +++ b/src/status_printer.cc @@ -0,0 +1,463 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "status_printer.h" + +#ifdef _WIN32 +#include "win32port.h" +#else +#ifndef __STDC_FORMAT_MACROS +#define __STDC_FORMAT_MACROS +#endif +#include +#endif + +#include +#include + +#ifdef _WIN32 +#include +#include +#endif + +#include "build.h" +#include "debug_flags.h" +#include "exit_status.h" + +using namespace std; + +Status* Status::factory(const BuildConfig& config) { + return new StatusPrinter(config); +} + +StatusPrinter::StatusPrinter(const BuildConfig& config) + : config_(config), started_edges_(0), finished_edges_(0), total_edges_(0), + running_edges_(0), progress_status_format_(NULL), + current_rate_(config.parallelism) { + // Don't do anything fancy in verbose mode. + if (config_.verbosity != BuildConfig::NORMAL) + printer_.set_smart_terminal(false); + + progress_status_format_ = getenv("NINJA_STATUS"); + if (!progress_status_format_) + progress_status_format_ = "[%f/%t] "; +} + +void StatusPrinter::EdgeAddedToPlan(const Edge* edge) { + ++total_edges_; + + // Do we know how long did this edge take last time? + if (edge->prev_elapsed_time_millis != -1) { + ++eta_predictable_edges_total_; + ++eta_predictable_edges_remaining_; + eta_predictable_cpu_time_total_millis_ += edge->prev_elapsed_time_millis; + eta_predictable_cpu_time_remaining_millis_ += + edge->prev_elapsed_time_millis; + } else + ++eta_unpredictable_edges_remaining_; +} + +void StatusPrinter::EdgeRemovedFromPlan(const Edge* edge) { + --total_edges_; + + // Do we know how long did this edge take last time? + if (edge->prev_elapsed_time_millis != -1) { + --eta_predictable_edges_total_; + --eta_predictable_edges_remaining_; + eta_predictable_cpu_time_total_millis_ -= edge->prev_elapsed_time_millis; + eta_predictable_cpu_time_remaining_millis_ -= + edge->prev_elapsed_time_millis; + } else + --eta_unpredictable_edges_remaining_; +} + +void StatusPrinter::BuildEdgeStarted(const Edge* edge, + int64_t start_time_millis) { + ++started_edges_; + ++running_edges_; + time_millis_ = start_time_millis; + + if (edge->use_console() || printer_.is_smart_terminal()) + PrintStatus(edge, start_time_millis); + + if (edge->use_console()) + printer_.SetConsoleLocked(true); +} + +void StatusPrinter::RecalculateProgressPrediction() { + time_predicted_percentage_ = 0.0; + + // Sometimes, the previous and actual times may be wildly different. + // For example, the previous build may have been fully recovered from ccache, + // so it was blazing fast, while the new build no longer gets hits from ccache + // for whatever reason, so it actually compiles code, which takes much longer. + // We should detect such cases, and avoid using "wrong" previous times. + + // Note that we will only use the previous times if there are edges with + // previous time knowledge remaining. + bool use_previous_times = eta_predictable_edges_remaining_ && + eta_predictable_cpu_time_remaining_millis_; + + // Iff we have sufficient statistical information for the current run, + // that is, if we have took at least 15 sec AND finished at least 5% of edges, + // we can check whether our performance so far matches the previous one. + if (use_previous_times && total_edges_ && finished_edges_ && + (time_millis_ >= 15 * 1e3) && + (((double)finished_edges_ / total_edges_) >= 0.05)) { + // Over the edges we've just run, how long did they take on average? + double actual_average_cpu_time_millis = + (double)cpu_time_millis_ / finished_edges_; + // What is the previous average, for the edges with such knowledge? + double previous_average_cpu_time_millis = + (double)eta_predictable_cpu_time_total_millis_ / + eta_predictable_edges_total_; + + double ratio = std::max(previous_average_cpu_time_millis, + actual_average_cpu_time_millis) / + std::min(previous_average_cpu_time_millis, + actual_average_cpu_time_millis); + + // Let's say that the average times should differ by less than 10x + use_previous_times = ratio < 10; + } + + int edges_with_known_runtime = finished_edges_; + if (use_previous_times) + edges_with_known_runtime += eta_predictable_edges_remaining_; + if (edges_with_known_runtime == 0) + return; + + int edges_with_unknown_runtime = use_previous_times + ? eta_unpredictable_edges_remaining_ + : (total_edges_ - finished_edges_); + + // Given the time elapsed on the edges we've just run, + // and the runtime of the edges for which we know previous runtime, + // what's the edge's average runtime? + int64_t edges_known_runtime_total_millis = cpu_time_millis_; + if (use_previous_times) + edges_known_runtime_total_millis += + eta_predictable_cpu_time_remaining_millis_; + + double average_cpu_time_millis = + (double)edges_known_runtime_total_millis / edges_with_known_runtime; + + // For the edges for which we do not have the previous runtime, + // let's assume that their average runtime is the same as for the other edges, + // and we therefore can predict their remaining runtime. + double unpredictable_cpu_time_remaining_millis = + average_cpu_time_millis * edges_with_unknown_runtime; + + // And therefore we can predict the remaining and total runtimes. + double total_cpu_time_remaining_millis = + unpredictable_cpu_time_remaining_millis; + if (use_previous_times) + total_cpu_time_remaining_millis += + eta_predictable_cpu_time_remaining_millis_; + double total_cpu_time_millis = + cpu_time_millis_ + total_cpu_time_remaining_millis; + if (total_cpu_time_millis == 0.0) + return; + + // After that we can tell how much work we've completed, in time units. + time_predicted_percentage_ = cpu_time_millis_ / total_cpu_time_millis; +} + +void StatusPrinter::BuildEdgeFinished(Edge* edge, int64_t start_time_millis, + int64_t end_time_millis, ExitStatus exit_code, + const string& output) { + time_millis_ = end_time_millis; + ++finished_edges_; + + int64_t elapsed = end_time_millis - start_time_millis; + cpu_time_millis_ += elapsed; + + // Do we know how long did this edge take last time? + if (edge->prev_elapsed_time_millis != -1) { + --eta_predictable_edges_remaining_; + eta_predictable_cpu_time_remaining_millis_ -= + edge->prev_elapsed_time_millis; + } else + --eta_unpredictable_edges_remaining_; + + if (edge->use_console()) + printer_.SetConsoleLocked(false); + + if (config_.verbosity == BuildConfig::QUIET) + return; + + if (!edge->use_console()) + PrintStatus(edge, end_time_millis); + + --running_edges_; + + // Print the command that is spewing before printing its output. + if (exit_code != ExitSuccess) { + string outputs; + for (vector::const_iterator o = edge->outputs_.begin(); + o != edge->outputs_.end(); ++o) + outputs += (*o)->path() + " "; + + string failed = "FAILED: [code=" + std::to_string(exit_code) + "] "; + if (printer_.supports_color()) { + printer_.PrintOnNewLine("\x1B[31m" + failed + "\x1B[0m" + outputs + "\n"); + } else { + printer_.PrintOnNewLine(failed + outputs + "\n"); + } + printer_.PrintOnNewLine(edge->EvaluateCommand() + "\n"); + } + + if (!output.empty()) { +#ifdef _WIN32 + // Fix extra CR being added on Windows, writing out CR CR LF (#773) + fflush(stdout); // Begin Windows extra CR fix + _setmode(_fileno(stdout), _O_BINARY); +#endif + + // ninja sets stdout and stderr of subprocesses to a pipe, to be able to + // check if the output is empty. Some compilers, e.g. clang, check + // isatty(stderr) to decide if they should print colored output. + // To make it possible to use colored output with ninja, subprocesses should + // be run with a flag that forces them to always print color escape codes. + // To make sure these escape codes don't show up in a file if ninja's output + // is piped to a file, ninja strips ansi escape codes again if it's not + // writing to a |smart_terminal_|. + // (Launching subprocesses in pseudo ttys doesn't work because there are + // only a few hundred available on some systems, and ninja can launch + // thousands of parallel compile commands.) + if (printer_.supports_color() || output.find('\x1b') == std::string::npos) { + printer_.PrintOnNewLine(output); + } else { + std::string final_output = StripAnsiEscapeCodes(output); + printer_.PrintOnNewLine(final_output); + } + +#ifdef _WIN32 + fflush(stdout); + _setmode(_fileno(stdout), _O_TEXT); // End Windows extra CR fix +#endif + } +} + +void StatusPrinter::BuildStarted() { + started_edges_ = 0; + finished_edges_ = 0; + running_edges_ = 0; +} + +void StatusPrinter::BuildFinished() { + printer_.SetConsoleLocked(false); + printer_.PrintOnNewLine(""); +} + +string StatusPrinter::FormatProgressStatus(const char* progress_status_format, + int64_t time_millis) const { + string out; + char buf[32]; + for (const char* s = progress_status_format; *s != '\0'; ++s) { + if (*s == '%') { + ++s; + switch (*s) { + case '%': + out.push_back('%'); + break; + + // Started edges. + case 's': + snprintf(buf, sizeof(buf), "%d", started_edges_); + out += buf; + break; + + // Total edges. + case 't': + snprintf(buf, sizeof(buf), "%d", total_edges_); + out += buf; + break; + + // Running edges. + case 'r': { + snprintf(buf, sizeof(buf), "%d", running_edges_); + out += buf; + break; + } + + // Unstarted edges. + case 'u': + snprintf(buf, sizeof(buf), "%d", total_edges_ - started_edges_); + out += buf; + break; + + // Finished edges. + case 'f': + snprintf(buf, sizeof(buf), "%d", finished_edges_); + out += buf; + break; + + // Overall finished edges per second. + case 'o': + SnprintfRate(finished_edges_ / (time_millis_ / 1e3), buf, "%.1f"); + out += buf; + break; + + // Current rate, average over the last '-j' jobs. + case 'c': + current_rate_.UpdateRate(finished_edges_, time_millis_); + SnprintfRate(current_rate_.rate(), buf, "%.1f"); + out += buf; + break; + + // Percentage of edges completed + case 'p': { + int percent = 0; + if (finished_edges_ != 0 && total_edges_ != 0) + percent = (100 * finished_edges_) / total_edges_; + snprintf(buf, sizeof(buf), "%3i%%", percent); + out += buf; + break; + } + +#define FORMAT_TIME_HMMSS(t) \ + "%" PRId64 ":%02" PRId64 ":%02" PRId64 "", (t) / 3600, ((t) % 3600) / 60, \ + (t) % 60 +#define FORMAT_TIME_MMSS(t) "%02" PRId64 ":%02" PRId64 "", (t) / 60, (t) % 60 + + // Wall time + case 'e': // elapsed, seconds + case 'w': // elapsed, human-readable + case 'E': // ETA, seconds + case 'W': // ETA, human-readable + { + double elapsed_sec = time_millis_ / 1e3; + double eta_sec = -1; // To be printed as "?". + if (time_predicted_percentage_ != 0.0) { + // So, we know that we've spent time_millis_ wall clock, + // and that is time_predicted_percentage_ percent. + // How much time will we need to complete 100%? + double total_wall_time = time_millis_ / time_predicted_percentage_; + // Naturally, that gives us the time remaining. + eta_sec = (total_wall_time - time_millis_) / 1e3; + } + + const bool print_with_hours = + elapsed_sec >= 60 * 60 || eta_sec >= 60 * 60; + + double sec = -1; + switch (*s) { + case 'e': // elapsed, seconds + case 'w': // elapsed, human-readable + sec = elapsed_sec; + break; + case 'E': // ETA, seconds + case 'W': // ETA, human-readable + sec = eta_sec; + break; + } + + if (sec < 0) + snprintf(buf, sizeof(buf), "?"); + else { + switch (*s) { + case 'e': // elapsed, seconds + case 'E': // ETA, seconds + snprintf(buf, sizeof(buf), "%.3f", sec); + break; + case 'w': // elapsed, human-readable + case 'W': // ETA, human-readable + if (print_with_hours) + snprintf(buf, sizeof(buf), FORMAT_TIME_HMMSS((int64_t)sec)); + else + snprintf(buf, sizeof(buf), FORMAT_TIME_MMSS((int64_t)sec)); + break; + } + } + out += buf; + break; + } + + // Percentage of time spent out of the predicted time total + case 'P': { + snprintf(buf, sizeof(buf), "%3i%%", + (int)(100. * time_predicted_percentage_)); + out += buf; + break; + } + + default: + Fatal("unknown placeholder '%%%c' in $NINJA_STATUS", *s); + return ""; + } + } else { + out.push_back(*s); + } + } + + return out; +} + +void StatusPrinter::PrintStatus(const Edge* edge, int64_t time_millis) { + if (explanations_) { + // Collect all explanations for the current edge's outputs. + std::vector explanations; + for (Node* output : edge->outputs_) { + explanations_->LookupAndAppend(output, &explanations); + } + if (!explanations.empty()) { + // Start a new line so that the first explanation does not append to the + // status line. + printer_.PrintOnNewLine(""); + for (const auto& exp : explanations) { + fprintf(stderr, "ninja explain: %s\n", exp.c_str()); + } + } + } + + if (config_.verbosity == BuildConfig::QUIET + || config_.verbosity == BuildConfig::NO_STATUS_UPDATE) + return; + + RecalculateProgressPrediction(); + + bool force_full_command = config_.verbosity == BuildConfig::VERBOSE; + + string to_print = edge->GetBinding("description"); + if (to_print.empty() || force_full_command) + to_print = edge->GetBinding("command"); + + to_print = FormatProgressStatus(progress_status_format_, time_millis) + + to_print; + + printer_.Print(to_print, + force_full_command ? LinePrinter::FULL : LinePrinter::ELIDE); +} + +void StatusPrinter::Warning(const char* msg, ...) { + va_list ap; + va_start(ap, msg); + ::Warning(msg, ap); + va_end(ap); +} + +void StatusPrinter::Error(const char* msg, ...) { + va_list ap; + va_start(ap, msg); + ::Error(msg, ap); + va_end(ap); +} + +void StatusPrinter::Info(const char* msg, ...) { + va_list ap; + va_start(ap, msg); + ::Info(msg, ap); + va_end(ap); +} diff --git a/src/status_printer.h b/src/status_printer.h new file mode 100644 index 0000000000..213e9ce825 --- /dev/null +++ b/src/status_printer.h @@ -0,0 +1,130 @@ +// Copyright 2016 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#pragma once + +#include +#include + +#include "exit_status.h" +#include "explanations.h" +#include "line_printer.h" +#include "status.h" + +/// Implementation of the Status interface that prints the status as +/// human-readable strings to stdout +struct StatusPrinter : Status { + explicit StatusPrinter(const BuildConfig& config); + + /// Callbacks for the Plan to notify us about adding/removing Edge's. + void EdgeAddedToPlan(const Edge* edge) override; + void EdgeRemovedFromPlan(const Edge* edge) override; + + void BuildEdgeStarted(const Edge* edge, int64_t start_time_millis) override; + void BuildEdgeFinished(Edge* edge, int64_t start_time_millis, + int64_t end_time_millis, ExitStatus exit_code, + const std::string& output) override; + void BuildStarted() override; + void BuildFinished() override; + + void Info(const char* msg, ...) override; + void Warning(const char* msg, ...) override; + void Error(const char* msg, ...) override; + + /// Format the progress status string by replacing the placeholders. + /// See the user manual for more information about the available + /// placeholders. + /// @param progress_status_format The format of the progress status. + /// @param status The status of the edge. + std::string FormatProgressStatus(const char* progress_status_format, + int64_t time_millis) const; + + /// Set the |explanations_| pointer. Used to implement `-d explain`. + void SetExplanations(Explanations* explanations) override { + explanations_ = explanations; + } + + private: + void PrintStatus(const Edge* edge, int64_t time_millis); + + const BuildConfig& config_; + + int started_edges_, finished_edges_, total_edges_, running_edges_; + + /// How much wall clock elapsed so far? + int64_t time_millis_ = 0; + + /// How much cpu clock elapsed so far? + int64_t cpu_time_millis_ = 0; + + /// What percentage of predicted total time have elapsed already? + double time_predicted_percentage_ = 0.0; + + /// Out of all the edges, for how many do we know previous time? + int eta_predictable_edges_total_ = 0; + /// And how much time did they all take? + int64_t eta_predictable_cpu_time_total_millis_ = 0; + + /// Out of all the non-finished edges, for how many do we know previous time? + int eta_predictable_edges_remaining_ = 0; + /// And how much time will they all take? + int64_t eta_predictable_cpu_time_remaining_millis_ = 0; + + /// For how many edges we don't know the previous run time? + int eta_unpredictable_edges_remaining_ = 0; + + void RecalculateProgressPrediction(); + + /// Prints progress output. + LinePrinter printer_; + + /// An optional Explanations pointer, used to implement `-d explain`. + Explanations* explanations_ = nullptr; + + /// The custom progress status format to use. + const char* progress_status_format_; + + template + void SnprintfRate(double rate, char (&buf)[S], const char* format) const { + if (rate == -1) + snprintf(buf, S, "?"); + else + snprintf(buf, S, format, rate); + } + + struct SlidingRateInfo { + SlidingRateInfo(int n) : rate_(-1), N(n), last_update_(-1) {} + + double rate() { return rate_; } + + void UpdateRate(int update_hint, int64_t time_millis) { + if (update_hint == last_update_) + return; + last_update_ = update_hint; + + if (times_.size() == N) + times_.pop(); + times_.push(time_millis); + if (times_.back() != times_.front()) + rate_ = times_.size() / ((times_.back() - times_.front()) / 1e3); + } + + private: + double rate_; + const size_t N; + std::queue times_; + int last_update_; + }; + + mutable SlidingRateInfo current_rate_; +}; diff --git a/src/status_test.cc b/src/status_test.cc new file mode 100644 index 0000000000..411d3ed03d --- /dev/null +++ b/src/status_test.cc @@ -0,0 +1,36 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "status.h" + +#include "test.h" + +TEST(StatusTest, StatusFormatElapsed) { + BuildConfig config; + StatusPrinter status(config); + + status.BuildStarted(); + // Before any task is done, the elapsed time must be zero. + EXPECT_EQ("[%/e0.000]", status.FormatProgressStatus("[%%/e%e]", 0)); + // Before any task is done, the elapsed time must be zero. + EXPECT_EQ("[%/e00:00]", status.FormatProgressStatus("[%%/e%w]", 0)); +} + +TEST(StatusTest, StatusFormatReplacePlaceholder) { + BuildConfig config; + StatusPrinter status(config); + + EXPECT_EQ("[%/s0/t0/r0/u0/f0]", + status.FormatProgressStatus("[%%/s%s/t%t/r%r/u%u/f%f]", 0)); +} diff --git a/src/string_piece.h b/src/string_piece.h index 031bda48d3..7e7367c20a 100644 --- a/src/string_piece.h +++ b/src/string_piece.h @@ -17,8 +17,6 @@ #include -using namespace std; - #include /// StringPiece represents a slice of a string whose memory is managed @@ -30,7 +28,7 @@ struct StringPiece { StringPiece() : str_(NULL), len_(0) {} /// The constructors intentionally allow for implicit conversions. - StringPiece(const string& str) : str_(str.data()), len_(str.size()) {} + StringPiece(const std::string& str) : str_(str.data()), len_(str.size()) {} StringPiece(const char* str) : str_(str), len_(strlen(str)) {} StringPiece(const char* str, size_t len) : str_(str), len_(len) {} @@ -38,14 +36,15 @@ struct StringPiece { bool operator==(const StringPiece& other) const { return len_ == other.len_ && memcmp(str_, other.str_, len_) == 0; } + bool operator!=(const StringPiece& other) const { return !(*this == other); } /// Convert the slice into a full-fledged std::string, copying the /// data into a new string. - string AsString() const { - return len_ ? string(str_, len_) : string(); + std::string AsString() const { + return len_ ? std::string(str_, len_) : std::string(); } const_iterator begin() const { @@ -64,6 +63,10 @@ struct StringPiece { return len_; } + size_t empty() const { + return len_ == 0; + } + const char* str_; size_t len_; }; diff --git a/src/string_piece_util.cc b/src/string_piece_util.cc index 8e1ecfddd7..69513f5dc7 100644 --- a/src/string_piece_util.cc +++ b/src/string_piece_util.cc @@ -39,7 +39,7 @@ vector SplitStringPiece(StringPiece input, char sep) { } string JoinStringPiece(const vector& list, char sep) { - if (list.size() == 0){ + if (list.empty()) { return ""; } diff --git a/src/string_piece_util.h b/src/string_piece_util.h index 2e40b9f3b2..28470f173e 100644 --- a/src/string_piece_util.h +++ b/src/string_piece_util.h @@ -19,11 +19,10 @@ #include #include "string_piece.h" -using namespace std; -vector SplitStringPiece(StringPiece input, char sep); +std::vector SplitStringPiece(StringPiece input, char sep); -string JoinStringPiece(const vector& list, char sep); +std::string JoinStringPiece(const std::vector& list, char sep); inline char ToLowerASCII(char c) { return (c >= 'A' && c <= 'Z') ? (c + ('a' - 'A')) : c; diff --git a/src/string_piece_util_test.cc b/src/string_piece_util_test.cc index 648c647918..cb296fed3c 100644 --- a/src/string_piece_util_test.cc +++ b/src/string_piece_util_test.cc @@ -16,12 +16,14 @@ #include "test.h" +using namespace std; + TEST(StringPieceUtilTest, SplitStringPiece) { { string input("a:b:c"); vector list = SplitStringPiece(input, ':'); - EXPECT_EQ(list.size(), 3); + EXPECT_EQ(list.size(), size_t(3)); EXPECT_EQ(list[0], "a"); EXPECT_EQ(list[1], "b"); @@ -29,10 +31,10 @@ TEST(StringPieceUtilTest, SplitStringPiece) { } { - string empty(""); + string empty; vector list = SplitStringPiece(empty, ':'); - EXPECT_EQ(list.size(), 1); + EXPECT_EQ(list.size(), size_t(1)); EXPECT_EQ(list[0], ""); } @@ -41,7 +43,7 @@ TEST(StringPieceUtilTest, SplitStringPiece) { string one("a"); vector list = SplitStringPiece(one, ':'); - EXPECT_EQ(list.size(), 1); + EXPECT_EQ(list.size(), size_t(1)); EXPECT_EQ(list[0], "a"); } @@ -50,7 +52,7 @@ TEST(StringPieceUtilTest, SplitStringPiece) { string sep_only(":"); vector list = SplitStringPiece(sep_only, ':'); - EXPECT_EQ(list.size(), 2); + EXPECT_EQ(list.size(), size_t(2)); EXPECT_EQ(list[0], ""); EXPECT_EQ(list[1], ""); @@ -60,7 +62,7 @@ TEST(StringPieceUtilTest, SplitStringPiece) { string sep(":a:b:c:"); vector list = SplitStringPiece(sep, ':'); - EXPECT_EQ(list.size(), 5); + EXPECT_EQ(list.size(), size_t(5)); EXPECT_EQ(list[0], ""); EXPECT_EQ(list[1], "a"); @@ -80,7 +82,7 @@ TEST(StringPieceUtilTest, JoinStringPiece) { } { - string empty(""); + string empty; vector list = SplitStringPiece(empty, ':'); EXPECT_EQ("", JoinStringPiece(list, ':')); diff --git a/src/subprocess-posix.cc b/src/subprocess-posix.cc index fc5543e85f..4d8da5606f 100644 --- a/src/subprocess-posix.cc +++ b/src/subprocess-posix.cc @@ -12,23 +12,31 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "exit_status.h" #include "subprocess.h" #include #include #include #include -#include #include #include #include #include #include +#if defined(USE_PPOLL) +#include +#else +#include +#endif + extern char** environ; #include "util.h" +using namespace std; + Subprocess::Subprocess(bool use_console) : fd_(-1), pid_(-1), use_console_(use_console) { } @@ -147,16 +155,27 @@ ExitStatus Subprocess::Finish() { Fatal("waitpid(%d): %s", pid_, strerror(errno)); pid_ = -1; +#ifdef _AIX + if (WIFEXITED(status) && WEXITSTATUS(status) & 0x80) { + // Map the shell's exit code used for signal failure (128 + signal) to the + // status code expected by AIX WIFSIGNALED and WTERMSIG macros which, unlike + // other systems, uses a different bit layout. + int signal = WEXITSTATUS(status) & 0x7f; + status = (signal << 16) | signal; + } +#endif + if (WIFEXITED(status)) { - int exit = WEXITSTATUS(status); - if (exit == 0) - return ExitSuccess; - } else if (WIFSIGNALED(status)) { + // propagate the status transparently + return static_cast(WEXITSTATUS(status)); + } + if (WIFSIGNALED(status)) { if (WTERMSIG(status) == SIGINT || WTERMSIG(status) == SIGTERM || WTERMSIG(status) == SIGHUP) return ExitInterrupted; } - return ExitFailure; + // At this point, we exit with any other signal+128 + return static_cast(status + 128); } bool Subprocess::Done() const { diff --git a/src/subprocess-win32.cc b/src/subprocess-win32.cc index a4a7669524..4cb8472141 100644 --- a/src/subprocess-win32.cc +++ b/src/subprocess-win32.cc @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "exit_status.h" #include "subprocess.h" #include @@ -21,6 +22,8 @@ #include "util.h" +using namespace std; + Subprocess::Subprocess(bool use_console) : child_(NULL) , overlapped_(), is_reading_(false), use_console_(use_console) { @@ -124,12 +127,20 @@ bool Subprocess::Start(SubprocessSet* set, const string& command) { buf_ = "CreateProcess failed: The system cannot find the file " "specified.\n"; return true; - } else if (error == ERROR_INVALID_PARAMETER) { - // This generally means that the command line was too long. Give extra - // context for this case. - Win32Fatal("CreateProcess", "is the command line too long?"); } else { - Win32Fatal("CreateProcess"); // pass all other errors to Win32Fatal + fprintf(stderr, "\nCreateProcess failed. Command attempted:\n\"%s\"\n", + command.c_str()); + const char* hint = NULL; + // ERROR_INVALID_PARAMETER means the command line was formatted + // incorrectly. This can be caused by a command line being too long or + // leading whitespace in the command. Give extra context for this case. + if (error == ERROR_INVALID_PARAMETER) { + if (command.length() > 0 && (command[0] == ' ' || command[0] == '\t')) + hint = "command contains leading whitespace"; + else + hint = "is the command line too long?"; + } + Win32Fatal("CreateProcess", hint); } } @@ -188,9 +199,8 @@ ExitStatus Subprocess::Finish() { CloseHandle(child_); child_ = NULL; - return exit_code == 0 ? ExitSuccess : - exit_code == CONTROL_C_EXIT ? ExitInterrupted : - ExitFailure; + return exit_code == CONTROL_C_EXIT ? ExitInterrupted : + static_cast(exit_code); } bool Subprocess::Done() const { diff --git a/src/subprocess.h b/src/subprocess.h index b2d486ca40..9e3d2ee98f 100644 --- a/src/subprocess.h +++ b/src/subprocess.h @@ -18,7 +18,6 @@ #include #include #include -using namespace std; #ifdef _WIN32 #include @@ -49,14 +48,14 @@ struct Subprocess { bool Done() const; - const string& GetOutput() const; + const std::string& GetOutput() const; private: Subprocess(bool use_console); - bool Start(struct SubprocessSet* set, const string& command); + bool Start(struct SubprocessSet* set, const std::string& command); void OnPipeReady(); - string buf_; + std::string buf_; #ifdef _WIN32 /// Set up pipe_ as the parent-side pipe of the subprocess; return the @@ -84,13 +83,13 @@ struct SubprocessSet { SubprocessSet(); ~SubprocessSet(); - Subprocess* Add(const string& command, bool use_console = false); + Subprocess* Add(const std::string& command, bool use_console = false); bool DoWork(); Subprocess* NextFinished(); void Clear(); - vector running_; - queue finished_; + std::vector running_; + std::queue finished_; #ifdef _WIN32 static BOOL WINAPI NotifyInterrupted(DWORD dwCtrlType); diff --git a/src/subprocess_test.cc b/src/subprocess_test.cc index 6e487dbde8..a1ece6dabd 100644 --- a/src/subprocess_test.cc +++ b/src/subprocess_test.cc @@ -14,6 +14,7 @@ #include "subprocess.h" +#include "exit_status.h" #include "test.h" #ifndef _WIN32 @@ -24,6 +25,8 @@ #include #endif +using namespace std; + namespace { #ifdef _WIN32 @@ -48,7 +51,8 @@ TEST_F(SubprocessTest, BadCommandStderr) { subprocs_.DoWork(); } - EXPECT_EQ(ExitFailure, subproc->Finish()); + ExitStatus exit = subproc->Finish(); + EXPECT_NE(ExitSuccess, exit); EXPECT_NE("", subproc->GetOutput()); } @@ -62,7 +66,8 @@ TEST_F(SubprocessTest, NoSuchCommand) { subprocs_.DoWork(); } - EXPECT_EQ(ExitFailure, subproc->Finish()); + ExitStatus exit = subproc->Finish(); + EXPECT_NE(ExitSuccess, exit); EXPECT_NE("", subproc->GetOutput()); #ifdef _WIN32 ASSERT_EQ("CreateProcess failed: The system cannot find the file " diff --git a/src/test.cc b/src/test.cc index a9816bc232..e9aaafa3bb 100644 --- a/src/test.cc +++ b/src/test.cc @@ -24,6 +24,7 @@ #include #ifdef _WIN32 #include +#include #else #include #endif @@ -33,22 +34,23 @@ #include "manifest_parser.h" #include "util.h" -namespace { - -#ifdef _WIN32 -#ifndef _mktemp_s -/// mingw has no mktemp. Implement one with the same type as the one -/// found in the Windows API. -int _mktemp_s(char* templ) { - char* ofs = strchr(templ, 'X'); - sprintf(ofs, "%d", rand() % 1000000); - return 0; +#ifdef _AIX +extern "C" { + // GCC "helpfully" strips the definition of mkdtemp out on AIX. + // The function is still present, so if we define it ourselves + // it will work perfectly fine. + extern char* mkdtemp(char* name_template); } #endif +using namespace std; + +namespace { + +#ifdef _WIN32 /// Windows has no mkdtemp. Implement it in terms of _mktemp_s. char* mkdtemp(char* name_template) { - int err = _mktemp_s(name_template); + int err = _mktemp_s(name_template, strlen(name_template) + 1); if (err < 0) { perror("_mktemp_s"); return NULL; @@ -233,3 +235,29 @@ void ScopedTempDir::Cleanup() { temp_dir_name_.clear(); } + +ScopedFilePath::ScopedFilePath(ScopedFilePath&& other) noexcept + : path_(std::move(other.path_)), released_(other.released_) { + other.released_ = true; +} + +/// It would be nice to use '= default' here instead but some old compilers +/// such as GCC from Ubuntu 16.06 will not compile it with "noexcept", so just +/// write it manually. +ScopedFilePath& ScopedFilePath::operator=(ScopedFilePath&& other) noexcept { + if (this != &other) { + this->~ScopedFilePath(); + new (this) ScopedFilePath(std::move(other)); + } + return *this; +} + +ScopedFilePath::~ScopedFilePath() { + if (!released_) { + platformAwareUnlink(path_.c_str()); + } +} + +void ScopedFilePath::Release() { + released_ = true; +} diff --git a/src/test.h b/src/test.h index 6af17b3f90..a4b9e19f75 100644 --- a/src/test.h +++ b/src/test.h @@ -15,94 +15,11 @@ #ifndef NINJA_TEST_H_ #define NINJA_TEST_H_ +#include + #include "disk_interface.h" #include "manifest_parser.h" #include "state.h" -#include "util.h" - -// A tiny testing framework inspired by googletest, but much simpler and -// faster to compile. It supports most things commonly used from googltest. The -// most noticeable things missing: EXPECT_* and ASSERT_* don't support -// streaming notes to them with operator<<, and for failing tests the lhs and -// rhs are not printed. That's so that this header does not have to include -// sstream, which slows down building ninja_test almost 20%. -namespace testing { -class Test { - bool failed_; - int assertion_failures_; - public: - Test() : failed_(false), assertion_failures_(0) {} - virtual ~Test() {} - virtual void SetUp() {} - virtual void TearDown() {} - virtual void Run() = 0; - - bool Failed() const { return failed_; } - int AssertionFailures() const { return assertion_failures_; } - void AddAssertionFailure() { assertion_failures_++; } - bool Check(bool condition, const char* file, int line, const char* error); -}; -} - -void RegisterTest(testing::Test* (*)(), const char*); - -extern testing::Test* g_current_test; -#define TEST_F_(x, y, name) \ - struct y : public x { \ - static testing::Test* Create() { return g_current_test = new y; } \ - virtual void Run(); \ - }; \ - struct Register##y { \ - Register##y() { RegisterTest(y::Create, name); } \ - }; \ - Register##y g_register_##y; \ - void y::Run() - -#define TEST_F(x, y) TEST_F_(x, x##y, #x "." #y) -#define TEST(x, y) TEST_F_(testing::Test, x##y, #x "." #y) - -#define EXPECT_EQ(a, b) \ - g_current_test->Check(a == b, __FILE__, __LINE__, #a " == " #b) -#define EXPECT_NE(a, b) \ - g_current_test->Check(a != b, __FILE__, __LINE__, #a " != " #b) -#define EXPECT_GT(a, b) \ - g_current_test->Check(a > b, __FILE__, __LINE__, #a " > " #b) -#define EXPECT_LT(a, b) \ - g_current_test->Check(a < b, __FILE__, __LINE__, #a " < " #b) -#define EXPECT_GE(a, b) \ - g_current_test->Check(a >= b, __FILE__, __LINE__, #a " >= " #b) -#define EXPECT_LE(a, b) \ - g_current_test->Check(a <= b, __FILE__, __LINE__, #a " <= " #b) -#define EXPECT_TRUE(a) \ - g_current_test->Check(static_cast(a), __FILE__, __LINE__, #a) -#define EXPECT_FALSE(a) \ - g_current_test->Check(!static_cast(a), __FILE__, __LINE__, #a) - -#define ASSERT_EQ(a, b) \ - if (!EXPECT_EQ(a, b)) { g_current_test->AddAssertionFailure(); return; } -#define ASSERT_NE(a, b) \ - if (!EXPECT_NE(a, b)) { g_current_test->AddAssertionFailure(); return; } -#define ASSERT_GT(a, b) \ - if (!EXPECT_GT(a, b)) { g_current_test->AddAssertionFailure(); return; } -#define ASSERT_LT(a, b) \ - if (!EXPECT_LT(a, b)) { g_current_test->AddAssertionFailure(); return; } -#define ASSERT_GE(a, b) \ - if (!EXPECT_GE(a, b)) { g_current_test->AddAssertionFailure(); return; } -#define ASSERT_LE(a, b) \ - if (!EXPECT_LE(a, b)) { g_current_test->AddAssertionFailure(); return; } -#define ASSERT_TRUE(a) \ - if (!EXPECT_TRUE(a)) { g_current_test->AddAssertionFailure(); return; } -#define ASSERT_FALSE(a) \ - if (!EXPECT_FALSE(a)) { g_current_test->AddAssertionFailure(); return; } -#define ASSERT_NO_FATAL_FAILURE(a) \ - { \ - int fail_count = g_current_test->AssertionFailures(); \ - a; \ - if (fail_count != g_current_test->AssertionFailures()) { \ - g_current_test->AddAssertionFailure(); \ - return; \ - } \ - } // Support utilities for tests. @@ -118,7 +35,7 @@ struct StateTestWithBuiltinRules : public testing::Test { void AddCatRule(State* state); /// Short way to get a Node by its path from state_. - Node* GetNode(const string& path); + Node* GetNode(const std::string& path); State state_; }; @@ -135,7 +52,7 @@ struct VirtualFileSystem : public DiskInterface { VirtualFileSystem() : now_(1) {} /// "Create" a file with contents. - void Create(const string& path, const string& contents); + void Create(const std::string& path, const std::string& contents); /// Tick "time" forwards; subsequent file operations will be newer than /// previous ones. @@ -144,25 +61,26 @@ struct VirtualFileSystem : public DiskInterface { } // DiskInterface - virtual TimeStamp Stat(const string& path, string* err) const; - virtual bool WriteFile(const string& path, const string& contents); - virtual bool MakeDir(const string& path); - virtual Status ReadFile(const string& path, string* contents, string* err); - virtual int RemoveFile(const string& path); + virtual TimeStamp Stat(const std::string& path, std::string* err) const; + virtual bool WriteFile(const std::string& path, const std::string& contents); + virtual bool MakeDir(const std::string& path); + virtual Status ReadFile(const std::string& path, std::string* contents, + std::string* err); + virtual int RemoveFile(const std::string& path); /// An entry for a single in-memory file. struct Entry { int mtime; - string stat_error; // If mtime is -1. - string contents; + std::string stat_error; // If mtime is -1. + std::string contents; }; - vector directories_made_; - vector files_read_; - typedef map FileMap; + std::vector directories_made_; + std::vector files_read_; + typedef std::map FileMap; FileMap files_; - set files_removed_; - set files_created_; + std::set files_removed_; + std::set files_created_; /// A simple fake timestamp for file operations. int now_; @@ -170,15 +88,42 @@ struct VirtualFileSystem : public DiskInterface { struct ScopedTempDir { /// Create a temporary directory and chdir into it. - void CreateAndEnter(const string& name); + void CreateAndEnter(const std::string& name); /// Clean up the temporary directory. void Cleanup(); /// The temp directory containing our dir. - string start_dir_; + std::string start_dir_; /// The subdirectory name for our dir, or empty if it hasn't been set up. - string temp_dir_name_; + std::string temp_dir_name_; +}; + +/// A class that records a file path and ensures that it is removed +/// on destruction. This ensures that tests do not keep stale files in the +/// current directory where they run, even in case of assertion failure. +struct ScopedFilePath { + /// Constructor just records the file path. + ScopedFilePath(const std::string& path) : path_(path) {} + ScopedFilePath(const char* path) : path_(path) {} + + /// Allow move operations. + ScopedFilePath(ScopedFilePath&&) noexcept; + ScopedFilePath& operator=(ScopedFilePath&&) noexcept; + + /// Destructor destroys the file, unless Release() was called. + ~ScopedFilePath(); + + /// Release the file, the destructor will not remove the file. + void Release(); + + const char* c_str() const { return path_.c_str(); } + const std::string& path() const { return path_; } + bool released() const { return released_; } + + private: + std::string path_; + bool released_ = false; }; #endif // NINJA_TEST_H_ diff --git a/src/third_party/emhash/README.ninja b/src/third_party/emhash/README.ninja new file mode 100644 index 0000000000..12ead4e545 --- /dev/null +++ b/src/third_party/emhash/README.ninja @@ -0,0 +1,8 @@ +Description: emhash8::HashMap for C++14/17 +Version: 1.6.5 (commit bdebddbdce1b473bbc189178fd523ef4a876ea01) +URL: https://github.com/ktprime/emhash +Copyright: Copyright (c) 2021-2024 Huang Yuanbing & bailuzhou AT 163.com +SPDX-License-Identifier: MIT +Local changes: + - Added includes for _mm_prefetch on MinGW. + - Fixed some spelling errors to appease the linter. diff --git a/src/third_party/emhash/hash_table8.hpp b/src/third_party/emhash/hash_table8.hpp new file mode 100644 index 0000000000..ec96e2d869 --- /dev/null +++ b/src/third_party/emhash/hash_table8.hpp @@ -0,0 +1,1834 @@ +// emhash8::HashMap for C++14/17 +// version 1.6.5 +// https://github.com/ktprime/emhash/blob/master/hash_table8.hpp +// +// Licensed under the MIT License . +// SPDX-License-Identifier: MIT +// Copyright (c) 2021-2024 Huang Yuanbing & bailuzhou AT 163.com +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#undef EMH_NEW +#undef EMH_EMPTY + +// likely/unlikely +#if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) +# define EMH_LIKELY(condition) __builtin_expect(condition, 1) +# define EMH_UNLIKELY(condition) __builtin_expect(condition, 0) +#else +# define EMH_LIKELY(condition) condition +# define EMH_UNLIKELY(condition) condition +#endif + +#define EMH_EMPTY(n) (0 > (int)(_index[n].next)) +#define EMH_EQHASH(n, key_hash) (((size_type)(key_hash) & ~_mask) == (_index[n].slot & ~_mask)) +//#define EMH_EQHASH(n, key_hash) ((size_type)(key_hash - _index[n].slot) & ~_mask) == 0 +#define EMH_NEW(key, val, bucket, key_hash) \ + new(_pairs + _num_filled) value_type(key, val); \ + _etail = bucket; \ + _index[bucket] = {bucket, _num_filled++ | ((size_type)(key_hash) & ~_mask)} + +#if _WIN32 && defined(_M_IX86) +#include +#endif + +namespace emhash8 { + +struct DefaultPolicy { + static constexpr float load_factor = 0.80f; + static constexpr float min_load_factor = 0.20f; + static constexpr size_t cacheline_size = 64U; +}; + +template, + typename EqT = std::equal_to, + typename Allocator = std::allocator>, //never used + typename Policy = DefaultPolicy> //never used +class HashMap +{ +#ifndef EMH_DEFAULT_LOAD_FACTOR + constexpr static float EMH_DEFAULT_LOAD_FACTOR = 0.80f; +#endif + constexpr static float EMH_MIN_LOAD_FACTOR = 0.25f; //< 0.5 + constexpr static uint32_t EMH_CACHE_LINE_SIZE = 64; //debug only + +public: + using htype = HashMap; + using value_type = std::pair; + using key_type = KeyT; + using mapped_type = ValueT; + //using dPolicy = Policy; + +#ifdef EMH_SMALL_TYPE + using size_type = uint16_t; +#elif EMH_SIZE_TYPE == 0 + using size_type = uint32_t; +#else + using size_type = size_t; +#endif + + using hasher = HashT; + using key_equal = EqT; + + constexpr static size_type INACTIVE = 0-1u; + //constexpr uint32_t END = 0-0x1u; + constexpr static size_type EAD = 2; + + struct Index + { + size_type next; + size_type slot; + }; + + class const_iterator; + class iterator + { + public: + using iterator_category = std::bidirectional_iterator_tag; + using difference_type = std::ptrdiff_t; + using value_type = typename htype::value_type; + using pointer = value_type*; + using const_pointer = const value_type* ; + using reference = value_type&; + using const_reference = const value_type&; + + iterator() : kv_(nullptr) {} + iterator(const_iterator& cit) { + kv_ = cit.kv_; + } + + iterator(const htype* hash_map, size_type bucket) { + kv_ = hash_map->_pairs + (int)bucket; + } + + iterator& operator++() + { + kv_ ++; + return *this; + } + + iterator operator++(int) + { + auto cur = *this; kv_ ++; + return cur; + } + + iterator& operator--() + { + kv_ --; + return *this; + } + + iterator operator--(int) + { + auto cur = *this; kv_ --; + return cur; + } + + reference operator*() const { return *kv_; } + pointer operator->() const { return kv_; } + + bool operator == (const iterator& rhs) const { return kv_ == rhs.kv_; } + bool operator != (const iterator& rhs) const { return kv_ != rhs.kv_; } + bool operator == (const const_iterator& rhs) const { return kv_ == rhs.kv_; } + bool operator != (const const_iterator& rhs) const { return kv_ != rhs.kv_; } + + public: + value_type* kv_; + }; + + class const_iterator + { + public: + using iterator_category = std::bidirectional_iterator_tag; + using value_type = typename htype::value_type; + using difference_type = std::ptrdiff_t; + using pointer = value_type*; + using const_pointer = const value_type*; + using reference = value_type&; + using const_reference = const value_type&; + + const_iterator(const iterator& it) { + kv_ = it.kv_; + } + + const_iterator (const htype* hash_map, size_type bucket) { + kv_ = hash_map->_pairs + (int)bucket; + } + + const_iterator& operator++() + { + kv_ ++; + return *this; + } + + const_iterator operator++(int) + { + auto cur = *this; kv_ ++; + return cur; + } + + const_iterator& operator--() + { + kv_ --; + return *this; + } + + const_iterator operator--(int) + { + auto cur = *this; kv_ --; + return cur; + } + + const_reference operator*() const { return *kv_; } + const_pointer operator->() const { return kv_; } + + bool operator == (const iterator& rhs) const { return kv_ == rhs.kv_; } + bool operator != (const iterator& rhs) const { return kv_ != rhs.kv_; } + bool operator == (const const_iterator& rhs) const { return kv_ == rhs.kv_; } + bool operator != (const const_iterator& rhs) const { return kv_ != rhs.kv_; } + public: + const value_type* kv_; + }; + + void init(size_type bucket, float mlf = EMH_DEFAULT_LOAD_FACTOR) + { + _pairs = nullptr; + _index = nullptr; + _mask = _num_buckets = 0; + _num_filled = 0; + _mlf = (uint32_t)((1 << 27) / EMH_DEFAULT_LOAD_FACTOR); + max_load_factor(mlf); + rehash(bucket); + } + + HashMap(size_type bucket = 2, float mlf = EMH_DEFAULT_LOAD_FACTOR) + { + init(bucket, mlf); + } + + HashMap(const HashMap& rhs) + { + if (rhs.load_factor() > EMH_MIN_LOAD_FACTOR) { + _pairs = alloc_bucket((size_type)(rhs._num_buckets * rhs.max_load_factor()) + 4); + _index = alloc_index(rhs._num_buckets); + clone(rhs); + } else { + init(rhs._num_filled + 2, rhs.max_load_factor()); + for (auto it = rhs.begin(); it != rhs.end(); ++it) + insert_unique(it->first, it->second); + } + } + + HashMap(HashMap&& rhs) noexcept + { + init(0); + *this = std::move(rhs); + } + + HashMap(std::initializer_list ilist) + { + init((size_type)ilist.size()); + for (auto it = ilist.begin(); it != ilist.end(); ++it) + do_insert(*it); + } + + template + HashMap(InputIt first, InputIt last, size_type bucket_count=4) + { + init(std::distance(first, last) + bucket_count); + for (; first != last; ++first) + emplace(*first); + } + + HashMap& operator=(const HashMap& rhs) + { + if (this == &rhs) + return *this; + + if (rhs.load_factor() < EMH_MIN_LOAD_FACTOR) { + clear(); free(_pairs); _pairs = nullptr; + rehash(rhs._num_filled + 2); + for (auto it = rhs.begin(); it != rhs.end(); ++it) + insert_unique(it->first, it->second); + return *this; + } + + clearkv(); + + if (_num_buckets != rhs._num_buckets) { + free(_pairs); free(_index); + _index = alloc_index(rhs._num_buckets); + _pairs = alloc_bucket((size_type)(rhs._num_buckets * rhs.max_load_factor()) + 4); + } + + clone(rhs); + return *this; + } + + HashMap& operator=(HashMap&& rhs) noexcept + { + if (this != &rhs) { + swap(rhs); + rhs.clear(); + } + return *this; + } + + template + bool operator == (const Con& rhs) const + { + if (size() != rhs.size()) + return false; + + for (auto it = begin(), last = end(); it != last; ++it) { + auto oi = rhs.find(it->first); + if (oi == rhs.end() || it->second != oi->second) + return false; + } + return true; + } + + template + bool operator != (const Con& rhs) const { return !(*this == rhs); } + + ~HashMap() noexcept + { + clearkv(); + free(_pairs); + free(_index); + _index = nullptr; + _pairs = nullptr; + } + + void clone(const HashMap& rhs) + { + _hasher = rhs._hasher; +// _eq = rhs._eq; + _num_buckets = rhs._num_buckets; + _num_filled = rhs._num_filled; + _mlf = rhs._mlf; + _last = rhs._last; + _mask = rhs._mask; +#if EMH_HIGH_LOAD + _ehead = rhs._ehead; +#endif + _etail = rhs._etail; + + auto opairs = rhs._pairs; + memcpy((char*)_index, (char*)rhs._index, (_num_buckets + EAD) * sizeof(Index)); + + if (is_copy_trivially()) { + memcpy((char*)_pairs, (char*)opairs, _num_filled * sizeof(value_type)); + } else { + for (size_type slot = 0; slot < _num_filled; slot++) + new(_pairs + slot) value_type(opairs[slot]); + } + } + + void swap(HashMap& rhs) + { + // std::swap(_eq, rhs._eq); + std::swap(_hasher, rhs._hasher); + std::swap(_pairs, rhs._pairs); + std::swap(_index, rhs._index); + std::swap(_num_buckets, rhs._num_buckets); + std::swap(_num_filled, rhs._num_filled); + std::swap(_mask, rhs._mask); + std::swap(_mlf, rhs._mlf); + std::swap(_last, rhs._last); +#if EMH_HIGH_LOAD + std::swap(_ehead, rhs._ehead); +#endif + std::swap(_etail, rhs._etail); + } + + // ------------------------------------------------------------- + iterator first() const { return {this, 0}; } + iterator last() const { return {this, _num_filled - 1}; } + + value_type& front() { return _pairs[0]; } + const value_type& front() const { return _pairs[0]; } + value_type& back() { return _pairs[_num_filled - 1]; } + const value_type& back() const { return _pairs[_num_filled - 1]; } + + void pop_front() { erase(begin()); } //TODO. only erase first without move last + void pop_back() { erase(last()); } + + iterator begin() { return first(); } + const_iterator cbegin() const { return first(); } + const_iterator begin() const { return first(); } + + iterator end() { return {this, _num_filled}; } + const_iterator cend() const { return {this, _num_filled}; } + const_iterator end() const { return cend(); } + + const value_type* values() const { return _pairs; } + const Index* index() const { return _index; } + + size_type size() const { return _num_filled; } + bool empty() const { return _num_filled == 0; } + size_type bucket_count() const { return _num_buckets; } + + /// Returns average number of elements per bucket. + float load_factor() const { return static_cast(_num_filled) / (_mask + 1); } + + HashT& hash_function() const { return _hasher; } + EqT& key_eq() const { return _eq; } + + void max_load_factor(float mlf) + { + if (mlf < 0.992 && mlf > EMH_MIN_LOAD_FACTOR) { + _mlf = (uint32_t)((1 << 27) / mlf); + if (_num_buckets > 0) rehash(_num_buckets); + } + } + + constexpr float max_load_factor() const { return (1 << 27) / (float)_mlf; } + constexpr size_type max_size() const { return (1ull << (sizeof(size_type) * 8 - 1)); } + constexpr size_type max_bucket_count() const { return max_size(); } + +#if EMH_STATIS + //Returns the bucket number where the element with key k is located. + size_type bucket(const KeyT& key) const + { + const auto bucket = hash_bucket(key); + const auto next_bucket = _index[bucket].next; + if ((int)next_bucket < 0) + return 0; + else if (bucket == next_bucket) + return bucket + 1; + + return hash_main(bucket) + 1; + } + + //Returns the number of elements in bucket n. + size_type bucket_size(const size_type bucket) const + { + auto next_bucket = _index[bucket].next; + if ((int)next_bucket < 0) + return 0; + + next_bucket = hash_main(bucket); + size_type ibucket_size = 1; + + //iterator each item in current main bucket + while (true) { + const auto nbucket = _index[next_bucket].next; + if (nbucket == next_bucket) { + break; + } + ibucket_size ++; + next_bucket = nbucket; + } + return ibucket_size; + } + + size_type get_main_bucket(const size_type bucket) const + { + auto next_bucket = _index[bucket].next; + if ((int)next_bucket < 0) + return INACTIVE; + + return hash_main(bucket); + } + + size_type get_diss(size_type bucket, size_type next_bucket, const size_type slots) const + { + auto pbucket = reinterpret_cast(&_pairs[bucket]); + auto pnext = reinterpret_cast(&_pairs[next_bucket]); + if (pbucket / EMH_CACHE_LINE_SIZE == pnext / EMH_CACHE_LINE_SIZE) + return 0; + size_type diff = pbucket > pnext ? (pbucket - pnext) : (pnext - pbucket); + if (diff / EMH_CACHE_LINE_SIZE < slots - 1) + return diff / EMH_CACHE_LINE_SIZE + 1; + return slots - 1; + } + + int get_bucket_info(const size_type bucket, size_type steps[], const size_type slots) const + { + auto next_bucket = _index[bucket].next; + if ((int)next_bucket < 0) + return -1; + + const auto main_bucket = hash_main(bucket); + if (next_bucket == main_bucket) + return 1; + else if (main_bucket != bucket) + return 0; + + steps[get_diss(bucket, next_bucket, slots)] ++; + size_type ibucket_size = 2; + //find a empty and linked it to tail + while (true) { + const auto nbucket = _index[next_bucket].next; + if (nbucket == next_bucket) + break; + + steps[get_diss(nbucket, next_bucket, slots)] ++; + ibucket_size ++; + next_bucket = nbucket; + } + return (int)ibucket_size; + } + + void dump_statics() const + { + const size_type slots = 128; + size_type buckets[slots + 1] = {0}; + size_type steps[slots + 1] = {0}; + for (size_type bucket = 0; bucket < _num_buckets; ++bucket) { + auto bsize = get_bucket_info(bucket, steps, slots); + if (bsize > 0) + buckets[bsize] ++; + } + + size_type sumb = 0, collision = 0, sumc = 0, finds = 0, sumn = 0; + puts("============== buckets size ration ========="); + for (size_type i = 0; i < sizeof(buckets) / sizeof(buckets[0]); i++) { + const auto bucketsi = buckets[i]; + if (bucketsi == 0) + continue; + sumb += bucketsi; + sumn += bucketsi * i; + collision += bucketsi * (i - 1); + finds += bucketsi * i * (i + 1) / 2; + printf(" %2u %8u %2.2lf| %.2lf\n", i, bucketsi, bucketsi * 100.0 * i / _num_filled, sumn * 100.0 / _num_filled); + } + + puts("========== collision miss ration ==========="); + for (size_type i = 0; i < sizeof(steps) / sizeof(steps[0]); i++) { + sumc += steps[i]; + if (steps[i] <= 2) + continue; + printf(" %2u %8u %.2lf %.2lf\n", i, steps[i], steps[i] * 100.0 / collision, sumc * 100.0 / collision); + } + + if (sumb == 0) return; + printf(" _num_filled/bucket_size/packed collision/cache_miss/hit_find = %u/%.2lf/%zd/ %.2lf%%/%.2lf%%/%.2lf\n", + _num_filled, _num_filled * 1.0 / sumb, sizeof(value_type), (collision * 100.0 / _num_filled), (collision - steps[0]) * 100.0 / _num_filled, finds * 1.0 / _num_filled); + assert(sumn == _num_filled); + assert(sumc == collision); + puts("============== buckets size end ============="); + } +#endif + + void pack_zero(ValueT zero) + { + _pairs[_num_filled] = {KeyT(), zero}; + } + + // ------------------------------------------------------------ + template + iterator find(const K& key) noexcept + { + return {this, find_filled_slot(key)}; + } + + template + const_iterator find(const K& key) const noexcept + { + return {this, find_filled_slot(key)}; + } + + template + ValueT& at(const K& key) + { + const auto slot = find_filled_slot(key); + //throw + return _pairs[slot].second; + } + + template + const ValueT& at(const K& key) const + { + const auto slot = find_filled_slot(key); + //throw + return _pairs[slot].second; + } + + const ValueT& index(const uint32_t index) const + { + return _pairs[index].second; + } + + ValueT& index(const uint32_t index) + { + return _pairs[index].second; + } + + template + bool contains(const K& key) const noexcept + { + return find_filled_slot(key) != _num_filled; + } + + template + size_type count(const K& key) const noexcept + { + return find_filled_slot(key) == _num_filled ? 0 : 1; + //return find_sorted_bucket(key) == END ? 0 : 1; + //return find_hash_bucket(key) == END ? 0 : 1; + } + + template + std::pair equal_range(const K& key) + { + const auto found = find(key); + if (found.second == _num_filled) + return { found, found }; + else + return { found, std::next(found) }; + } + + void merge(HashMap& rhs) + { + if (empty()) { + *this = std::move(rhs); + return; + } + + for (auto rit = rhs.begin(); rit != rhs.end(); ) { + auto fit = find(rit->first); + if (fit == end()) { + insert_unique(rit->first, std::move(rit->second)); + rit = rhs.erase(rit); + } else { + ++rit; + } + } + } + + /// Returns the matching ValueT or nullptr if k isn't found. + bool try_get(const KeyT& key, ValueT& val) const noexcept + { + const auto slot = find_filled_slot(key); + const auto found = slot != _num_filled; + if (found) { + val = _pairs[slot].second; + } + return found; + } + + /// Returns the matching ValueT or nullptr if k isn't found. + ValueT* try_get(const KeyT& key) noexcept + { + const auto slot = find_filled_slot(key); + return slot != _num_filled ? &_pairs[slot].second : nullptr; + } + + /// Const version of the above + ValueT* try_get(const KeyT& key) const noexcept + { + const auto slot = find_filled_slot(key); + return slot != _num_filled ? &_pairs[slot].second : nullptr; + } + + /// set value if key exist + bool try_set(const KeyT& key, const ValueT& val) noexcept + { + const auto slot = find_filled_slot(key); + if (slot == _num_filled) + return false; + + _pairs[slot].second = val; + return true; + } + + /// set value if key exist + bool try_set(const KeyT& key, ValueT&& val) noexcept + { + const auto slot = find_filled_slot(key); + if (slot == _num_filled) + return false; + + _pairs[slot].second = std::move(val); + return true; + } + + /// Convenience function. + ValueT get_or_return_default(const KeyT& key) const noexcept + { + const auto slot = find_filled_slot(key); + return slot == _num_filled ? ValueT() : _pairs[slot].second; + } + + // ----------------------------------------------------- + std::pair do_insert(const value_type& value) noexcept + { + const auto key_hash = hash_key(value.first); + const auto bucket = find_or_allocate(value.first, key_hash); + const auto bempty = EMH_EMPTY(bucket); + if (bempty) { + EMH_NEW(value.first, value.second, bucket, key_hash); + } + + const auto slot = _index[bucket].slot & _mask; + return { {this, slot}, bempty }; + } + + std::pair do_insert(value_type&& value) noexcept + { + const auto key_hash = hash_key(value.first); + const auto bucket = find_or_allocate(value.first, key_hash); + const auto bempty = EMH_EMPTY(bucket); + if (bempty) { + EMH_NEW(std::move(value.first), std::move(value.second), bucket, key_hash); + } + + const auto slot = _index[bucket].slot & _mask; + return { {this, slot}, bempty }; + } + + template + std::pair do_insert(K&& key, V&& val) noexcept + { + const auto key_hash = hash_key(key); + const auto bucket = find_or_allocate(key, key_hash); + const auto bempty = EMH_EMPTY(bucket); + if (bempty) { + EMH_NEW(std::forward(key), std::forward(val), bucket, key_hash); + } + + const auto slot = _index[bucket].slot & _mask; + return { {this, slot}, bempty }; + } + + template + std::pair do_assign(K&& key, V&& val) noexcept + { + check_expand_need(); + const auto key_hash = hash_key(key); + const auto bucket = find_or_allocate(key, key_hash); + const auto bempty = EMH_EMPTY(bucket); + if (bempty) { + EMH_NEW(std::forward(key), std::forward(val), bucket, key_hash); + } else { + _pairs[_index[bucket].slot & _mask].second = std::move(val); + } + + const auto slot = _index[bucket].slot & _mask; + return { {this, slot}, bempty }; + } + + std::pair insert(const value_type& p) + { + check_expand_need(); + return do_insert(p); + } + + std::pair insert(value_type && p) + { + check_expand_need(); + return do_insert(std::move(p)); + } + + void insert(std::initializer_list ilist) + { + reserve(ilist.size() + _num_filled, false); + for (auto it = ilist.begin(); it != ilist.end(); ++it) + do_insert(*it); + } + + template + void insert(Iter first, Iter last) + { + reserve(std::distance(first, last) + _num_filled, false); + for (; first != last; ++first) + do_insert(first->first, first->second); + } + +#if 0 + template + void insert_unique(Iter begin, Iter end) + { + reserve(std::distance(begin, end) + _num_filled, false); + for (; begin != end; ++begin) { + insert_unique(*begin); + } + } +#endif + + template + size_type insert_unique(K&& key, V&& val) + { + check_expand_need(); + const auto key_hash = hash_key(key); + auto bucket = find_unique_bucket(key_hash); + EMH_NEW(std::forward(key), std::forward(val), bucket, key_hash); + return bucket; + } + + size_type insert_unique(value_type&& value) + { + return insert_unique(std::move(value.first), std::move(value.second)); + } + + size_type insert_unique(const value_type& value) + { + return insert_unique(value.first, value.second); + } + + template + std::pair emplace(Args&&... args) noexcept + { + check_expand_need(); + return do_insert(std::forward(args)...); + } + + //no any optimize for position + template + iterator emplace_hint(const_iterator hint, Args&&... args) + { + (void)hint; + check_expand_need(); + return do_insert(std::forward(args)...).first; + } + + template + std::pair try_emplace(const KeyT& k, Args&&... args) + { + check_expand_need(); + return do_insert(k, std::forward(args)...); + } + + template + std::pair try_emplace(KeyT&& k, Args&&... args) + { + check_expand_need(); + return do_insert(std::move(k), std::forward(args)...); + } + + template + size_type emplace_unique(Args&&... args) + { + return insert_unique(std::forward(args)...); + } + + std::pair insert_or_assign(const KeyT& key, ValueT&& val) { return do_assign(key, std::forward(val)); } + std::pair insert_or_assign(KeyT&& key, ValueT&& val) { return do_assign(std::move(key), std::forward(val)); } + + /// Return the old value or ValueT() if it didn't exist. + ValueT set_get(const KeyT& key, const ValueT& val) + { + check_expand_need(); + const auto key_hash = hash_key(key); + const auto bucket = find_or_allocate(key, key_hash); + if (EMH_EMPTY(bucket)) { + EMH_NEW(key, val, bucket, key_hash); + return ValueT(); + } else { + const auto slot = _index[bucket].slot & _mask; + ValueT old_value(val); + std::swap(_pairs[slot].second, old_value); + return old_value; + } + } + + /// Like std::map::operator[]. + ValueT& operator[](const KeyT& key) noexcept + { + check_expand_need(); + const auto key_hash = hash_key(key); + const auto bucket = find_or_allocate(key, key_hash); + if (EMH_EMPTY(bucket)) { + /* Check if inserting a value rather than overwriting an old entry */ + EMH_NEW(key, std::move(ValueT()), bucket, key_hash); + } + + const auto slot = _index[bucket].slot & _mask; + return _pairs[slot].second; + } + + ValueT& operator[](KeyT&& key) noexcept + { + check_expand_need(); + const auto key_hash = hash_key(key); + const auto bucket = find_or_allocate(key, key_hash); + if (EMH_EMPTY(bucket)) { + EMH_NEW(std::move(key), std::move(ValueT()), bucket, key_hash); + } + + const auto slot = _index[bucket].slot & _mask; + return _pairs[slot].second; + } + + /// Erase an element from the hash table. + /// return 0 if element was not found + size_type erase(const KeyT& key) noexcept + { + const auto key_hash = hash_key(key); + const auto sbucket = find_filled_bucket(key, key_hash); + if (sbucket == INACTIVE) + return 0; + + const auto main_bucket = key_hash & _mask; + erase_slot(sbucket, (size_type)main_bucket); + return 1; + } + + //iterator erase(const_iterator begin_it, const_iterator end_it) + iterator erase(const const_iterator& cit) noexcept + { + const auto slot = (size_type)(cit.kv_ - _pairs); + size_type main_bucket; + const auto sbucket = find_slot_bucket(slot, main_bucket); //TODO + erase_slot(sbucket, main_bucket); + return {this, slot}; + } + + //only last >= first + iterator erase(const_iterator first, const_iterator last) noexcept + { + auto esize = long(last.kv_ - first.kv_); + auto tsize = long((_pairs + _num_filled) - last.kv_); //last to tail size + auto next = first; + while (tsize -- > 0) { + if (esize-- <= 0) + break; + next = ++erase(next); + } + + //fast erase from last + next = this->last(); + while (esize -- > 0) + next = --erase(next); + + return {this, size_type(next.kv_ - _pairs)}; + } + + template + size_type erase_if(Pred pred) + { + auto old_size = size(); + for (auto it = begin(); it != end();) { + if (pred(*it)) + it = erase(it); + else + ++it; + } + return old_size - size(); + } + + static constexpr bool is_triviall_destructable() + { +#if __cplusplus >= 201402L || _MSC_VER > 1600 + return !(std::is_trivially_destructible::value && std::is_trivially_destructible::value); +#else + return !(std::is_pod::value && std::is_pod::value); +#endif + } + + static constexpr bool is_copy_trivially() + { +#if __cplusplus >= 201103L || _MSC_VER > 1600 + return (std::is_trivially_copyable::value && std::is_trivially_copyable::value); +#else + return (std::is_pod::value && std::is_pod::value); +#endif + } + + void clearkv() + { + if (is_triviall_destructable()) { + while (_num_filled --) + _pairs[_num_filled].~value_type(); + } + } + + /// Remove all elements, keeping full capacity. + void clear() noexcept + { + clearkv(); + + if (_num_filled > 0) + memset((char*)_index, INACTIVE, sizeof(_index[0]) * _num_buckets); + + _last = _num_filled = 0; + _etail = INACTIVE; + +#if EMH_HIGH_LOAD + _ehead = 0; +#endif + } + + void shrink_to_fit(const float min_factor = EMH_DEFAULT_LOAD_FACTOR / 4) + { + if (load_factor() < min_factor && bucket_count() > 10) //safe guard + rehash(_num_filled + 1); + } + +#if EMH_HIGH_LOAD + #define EMH_PREVET(i, n) i[n].slot + void set_empty() + { + auto prev = 0; + for (int32_t bucket = 1; bucket < _num_buckets; ++bucket) { + if (EMH_EMPTY(bucket)) { + if (prev != 0) { + EMH_PREVET(_index, bucket) = prev; + _index[_prev].next = -bucket; + } + else + _ehead = bucket; + prev = bucket; + } + } + + EMH_PREVET(_index, _ehead) = prev; + _index[_prev].next = 0-_ehead; + _ehead = 0-_index[_ehead].next; + } + + void clear_empty() + { + auto prev = EMH_PREVET(_index, _ehead); + while (prev != _ehead) { + _index[_prev].next = INACTIVE; + prev = EMH_PREVET(_index, prev); + } + _index[_ehead].next = INACTIVE; + _ehead = 0; + } + + //prev-ehead->next + size_type pop_empty(const size_type bucket) + { + const auto prev_bucket = EMH_PREVET(_index, bucket); + const int next_bucket = 0-_index[bucket].next; + + EMH_PREVET(_index, next_bucket) = prev_bucket; + _index[prev_bucket].next = -next_bucket; + + _ehead = next_bucket; + return bucket; + } + + //ehead->bucket->next + void push_empty(const int32_t bucket) + { + const int next_bucket = 0-_index[_ehead].next; + assert(next_bucket > 0); + + EMH_PREVET(_index, bucket) = _ehead; + _index[bucket].next = -next_bucket; + + EMH_PREVET(_index, next_bucket) = bucket; + _index[_ehead].next = -bucket; + // _ehead = bucket; + } +#endif + + /// Make room for this many elements + bool reserve(uint64_t num_elems, bool force) + { + (void)force; +#if EMH_HIGH_LOAD == 0 + const auto required_buckets = num_elems * _mlf >> 27; + if (EMH_LIKELY(required_buckets < _mask)) // && !force + return false; + +#elif EMH_HIGH_LOAD + const auto required_buckets = num_elems + num_elems * 1 / 9; + if (EMH_LIKELY(required_buckets < _mask)) + return false; + + else if (_num_buckets < 16 && _num_filled < _num_buckets) + return false; + + else if (_num_buckets > EMH_HIGH_LOAD) { + if (_ehead == 0) { + set_empty(); + return false; + } else if (/*_num_filled + 100 < _num_buckets && */_index[_ehead].next != 0-_ehead) { + return false; + } + } +#endif +#if EMH_STATIS + if (_num_filled > EMH_STATIS) dump_statics(); +#endif + + //assert(required_buckets < max_size()); + rehash(required_buckets + 2); + return true; + } + + static value_type* alloc_bucket(size_type num_buckets) + { +#ifdef EMH_ALLOC + auto new_pairs = aligned_alloc(32, (uint64_t)num_buckets * sizeof(value_type)); +#else + auto new_pairs = malloc((uint64_t)num_buckets * sizeof(value_type)); +#endif + return (value_type *)(new_pairs); + } + + static Index* alloc_index(size_type num_buckets) + { + auto new_index = (char*)malloc((uint64_t)(EAD + num_buckets) * sizeof(Index)); + return (Index *)(new_index); + } + + bool reserve(size_type required_buckets) noexcept + { + if (_num_filled != required_buckets) + return reserve(required_buckets, true); + + _last = 0; +#if EMH_HIGH_LOAD + _ehead = 0; +#endif + +#if EMH_SORT + std::sort(_pairs, _pairs + _num_filled, [this](const value_type & l, const value_type & r) { + const auto hashl = (size_type)hash_key(l.first) & _mask, hashr = (size_type)hash_key(r.first) & _mask; + return hashl < hashr; + //return l.first < r.first; + }); +#endif + + memset((char*)_index, INACTIVE, sizeof(_index[0]) * _num_buckets); + for (size_type slot = 0; slot < _num_filled; slot++) { + const auto& key = _pairs[slot].first; + const auto key_hash = hash_key(key); + const auto bucket = size_type(key_hash & _mask); + auto& next_bucket = _index[bucket].next; + if ((int)next_bucket < 0) + _index[bucket] = {1, slot | ((size_type)(key_hash) & ~_mask)}; + else { + _index[bucket].slot |= (size_type)(key_hash) & ~_mask; + next_bucket ++; + } + } + return true; + } + + void rebuild(size_type num_buckets) noexcept + { + free(_index); + auto new_pairs = (value_type*)alloc_bucket((size_type)(num_buckets * max_load_factor()) + 4); + if (is_copy_trivially()) { + if (_pairs) + memcpy((char*)new_pairs, (char*)_pairs, _num_filled * sizeof(value_type)); + } else { + for (size_type slot = 0; slot < _num_filled; slot++) { + new(new_pairs + slot) value_type(std::move(_pairs[slot])); + if (is_triviall_destructable()) + _pairs[slot].~value_type(); + } + } + free(_pairs); + _pairs = new_pairs; + _index = (Index*)alloc_index (num_buckets); + + memset((char*)_index, INACTIVE, sizeof(_index[0]) * num_buckets); + memset((char*)(_index + num_buckets), 0, sizeof(_index[0]) * EAD); + } + + void rehash(uint64_t required_buckets) + { + if (required_buckets < _num_filled) + return; + + assert(required_buckets < max_size()); + auto num_buckets = _num_filled > (1u << 16) ? (1u << 16) : 4u; + while (num_buckets < required_buckets) { num_buckets *= 2; } +#if EMH_SAVE_MEM + if (sizeof(KeyT) < sizeof(size_type) && num_buckets >= (1ul << (2 * 8))) + num_buckets = 2ul << (sizeof(KeyT) * 8); +#endif + +#if EMH_REHASH_LOG + auto last = _last; + size_type collision = 0; +#endif + +#if EMH_HIGH_LOAD + _ehead = 0; +#endif + _last = 0; + + _mask = num_buckets - 1; +#if EMH_PACK_TAIL > 1 + _last = _mask; + num_buckets += num_buckets * EMH_PACK_TAIL / 100; //add more 5-10% +#endif + _num_buckets = num_buckets; + + rebuild(num_buckets); + +#ifdef EMH_SORT + std::sort(_pairs, _pairs + _num_filled, [this](const value_type & l, const value_type & r) { + const auto hashl = hash_key(l.first), hashr = hash_key(r.first); + auto diff = int64_t((hashl & _mask) - (hashr & _mask)); + if (diff != 0) + return diff < 0; + return hashl < hashr; +// return l.first < r.first; + }); +#endif + + _etail = INACTIVE; + for (size_type slot = 0; slot < _num_filled; ++slot) { + const auto& key = _pairs[slot].first; + const auto key_hash = hash_key(key); + const auto bucket = find_unique_bucket(key_hash); + _index[bucket] = { bucket, slot | ((size_type)(key_hash) & ~_mask) }; + +#if EMH_REHASH_LOG + if (bucket != hash_main(bucket)) + collision ++; +#endif + } + +#if EMH_REHASH_LOG + if (_num_filled > EMH_REHASH_LOG) { + auto mbucket = _num_filled - collision; + char buff[255] = {0}; + sprintf(buff, " _num_filled/aver_size/K.V/pack/collision|last = %u/%.2lf/%s.%s/%zd|%.2lf%%,%.2lf%%", + _num_filled, double (_num_filled) / mbucket, typeid(KeyT).name(), typeid(ValueT).name(), sizeof(_pairs[0]), collision * 100.0 / _num_filled, last * 100.0 / _num_buckets); +#ifdef EMH_LOG + static uint32_t ihashs = 0; EMH_LOG() << "hash_nums = " << ihashs ++ << "|" <<__FUNCTION__ << "|" << buff << endl; +#else + puts(buff); +#endif + } +#endif + } + +private: + // Can we fit another element? + bool check_expand_need() + { + return reserve(_num_filled, false); + } + + static void prefetch_heap_block(char* ctrl) + { + // Prefetch the heap-allocated memory region to resolve potential TLB + // misses. This is intended to overlap with execution of calculating the hash for a key. +#if __linux__ + __builtin_prefetch(static_cast(ctrl)); +#elif _WIN32 && defined(_M_IX86) + _mm_prefetch((const char*)ctrl, _MM_HINT_T0); +#endif + } + + size_type slot_to_bucket(const size_type slot) const noexcept + { + size_type main_bucket; + return find_slot_bucket(slot, main_bucket); //TODO + } + + //very slow + void erase_slot(const size_type sbucket, const size_type main_bucket) noexcept + { + const auto slot = _index[sbucket].slot & _mask; + const auto ebucket = erase_bucket(sbucket, main_bucket); + const auto last_slot = --_num_filled; + if (EMH_LIKELY(slot != last_slot)) { + const auto last_bucket = (_etail == INACTIVE || ebucket == _etail) + ? slot_to_bucket(last_slot) : _etail; + + _pairs[slot] = std::move(_pairs[last_slot]); + _index[last_bucket].slot = slot | (_index[last_bucket].slot & ~_mask); + } + + if (is_triviall_destructable()) + _pairs[last_slot].~value_type(); + + _etail = INACTIVE; + _index[ebucket] = {INACTIVE, 0}; +#if EMH_HIGH_LOAD + if (_ehead) { + if (10 * _num_filled < 8 * _num_buckets) + clear_empty(); + else if (ebucket) + push_empty(ebucket); + } +#endif + } + + size_type erase_bucket(const size_type bucket, const size_type main_bucket) noexcept + { + const auto next_bucket = _index[bucket].next; + if (bucket == main_bucket) { + if (main_bucket != next_bucket) { + const auto nbucket = _index[next_bucket].next; + _index[main_bucket] = { + (nbucket == next_bucket) ? main_bucket : nbucket, + _index[next_bucket].slot + }; + } + return next_bucket; + } + + const auto prev_bucket = find_prev_bucket(main_bucket, bucket); + _index[prev_bucket].next = (bucket == next_bucket) ? prev_bucket : next_bucket; + return bucket; + } + + // Find the slot with this key, or return bucket size + size_type find_slot_bucket(const size_type slot, size_type& main_bucket) const + { + const auto key_hash = hash_key(_pairs[slot].first); + const auto bucket = main_bucket = size_type(key_hash & _mask); + if (slot == (_index[bucket].slot & _mask)) + return bucket; + + auto next_bucket = _index[bucket].next; + while (true) { + if (EMH_LIKELY(slot == (_index[next_bucket].slot & _mask))) + return next_bucket; + next_bucket = _index[next_bucket].next; + } + + return INACTIVE; + } + + // Find the slot with this key, or return bucket size + size_type find_filled_bucket(const KeyT& key, uint64_t key_hash) const noexcept + { + const auto bucket = size_type(key_hash & _mask); + auto next_bucket = _index[bucket].next; + if (EMH_UNLIKELY((int)next_bucket < 0)) + return INACTIVE; + + const auto slot = _index[bucket].slot & _mask; + //prefetch_heap_block((char*)&_pairs[slot]); + if (EMH_EQHASH(bucket, key_hash)) { + if (EMH_LIKELY(_eq(key, _pairs[slot].first))) + return bucket; + } + if (next_bucket == bucket) + return INACTIVE; + + while (true) { + if (EMH_EQHASH(next_bucket, key_hash)) { + const auto next_slot = _index[next_bucket].slot & _mask; + if (EMH_LIKELY(_eq(key, _pairs[next_slot].first))) + return next_bucket; + } + + const auto nbucket = _index[next_bucket].next; + if (nbucket == next_bucket) + return INACTIVE; + next_bucket = nbucket; + } + + return INACTIVE; + } + + // Find the slot with this key, or return bucket size + template + size_type find_filled_slot(const K& key) const noexcept + { + const auto key_hash = hash_key(key); + const auto bucket = size_type(key_hash & _mask); + auto next_bucket = _index[bucket].next; + if ((int)next_bucket < 0) + return _num_filled; + + const auto slot = _index[bucket].slot & _mask; + //prefetch_heap_block((char*)&_pairs[slot]); + if (EMH_EQHASH(bucket, key_hash)) { + if (EMH_LIKELY(_eq(key, _pairs[slot].first))) + return slot; + } + if (next_bucket == bucket) + return _num_filled; + + while (true) { + if (EMH_EQHASH(next_bucket, key_hash)) { + const auto next_slot = _index[next_bucket].slot & _mask; + if (EMH_LIKELY(_eq(key, _pairs[next_slot].first))) + return next_slot; + } + + const auto nbucket = _index[next_bucket].next; + if (nbucket == next_bucket) + return _num_filled; + next_bucket = nbucket; + } + + return _num_filled; + } + +#if EMH_SORT + size_type find_hash_bucket(const KeyT& key) const noexcept + { + const auto key_hash = hash_key(key); + const auto bucket = size_type(key_hash & _mask); + const auto next_bucket = _index[bucket].next; + if ((int)next_bucket < 0) + return END; + + auto slot = _index[bucket].slot & _mask; + if (_eq(key, _pairs[slot++].first)) + return slot; + else if (next_bucket == bucket) + return END; + + while (true) { + const auto& okey = _pairs[slot++].first; + if (_eq(key, okey)) + return slot; + + const auto hasho = hash_key(okey); + if ((hasho & _mask) != bucket) + break; + else if (hasho > key_hash) + break; + else if (EMH_UNLIKELY(slot >= _num_filled)) + break; + } + + return END; + } + + //only for find/can not insert + size_type find_sorted_bucket(const KeyT& key) const noexcept + { + const auto key_hash = hash_key(key); + const auto bucket = size_type(key_hash & _mask); + const auto slots = (int)(_index[bucket].next); //TODO + if (slots < 0 /**|| key < _pairs[slot].first*/) + return END; + + const auto slot = _index[bucket].slot & _mask; + auto ormask = _index[bucket].slot & ~_mask; + auto hmask = (size_type)(key_hash) & ~_mask; + if ((hmask | ormask) != ormask) + return END; + + if (_eq(key, _pairs[slot].first)) + return slot; + else if (slots == 1 || key < _pairs[slot].first) + return END; + +#if EMH_SORT + if (key < _pairs[slot].first || key > _pairs[slots + slot - 1].first) + return END; +#endif + + for (size_type i = 1; i < slots; ++i) { + const auto& okey = _pairs[slot + i].first; + if (_eq(key, okey)) + return slot + i; + // else if (okey > key) + // return END; + } + + return END; + } +#endif + + //kick out bucket and find empty to occpuy + //it will break the origin link and relink again. + //before: main_bucket-->prev_bucket --> bucket --> next_bucket + //after : main_bucket-->prev_bucket --> (removed)--> new_bucket--> next_bucket + size_type kickout_bucket(const size_type kmain, const size_type bucket) noexcept + { + const auto next_bucket = _index[bucket].next; + const auto new_bucket = find_empty_bucket(next_bucket, 2); + const auto prev_bucket = find_prev_bucket(kmain, bucket); + + const auto last = next_bucket == bucket ? new_bucket : next_bucket; + _index[new_bucket] = {last, _index[bucket].slot}; + + _index[prev_bucket].next = new_bucket; + _index[bucket].next = INACTIVE; + + return bucket; + } + + /* + ** inserts a new key into a hash table; first, check whether key's main + ** bucket/position is free. If not, check whether colliding node/bucket is in its main + ** position or not: if it is not, move colliding bucket to an empty place and + ** put new key in its main position; otherwise (colliding bucket is in its main + ** position), new key goes to an empty position. + */ + template + size_type find_or_allocate(const K& key, uint64_t key_hash) noexcept + { + const auto bucket = size_type(key_hash & _mask); + auto next_bucket = _index[bucket].next; + prefetch_heap_block((char*)&_pairs[bucket]); + if ((int)next_bucket < 0) { +#if EMH_HIGH_LOAD + if (next_bucket != INACTIVE) + pop_empty(bucket); +#endif + return bucket; + } + + const auto slot = _index[bucket].slot & _mask; + if (EMH_EQHASH(bucket, key_hash)) + if (EMH_LIKELY(_eq(key, _pairs[slot].first))) + return bucket; + + //check current bucket_key is in main bucket or not + const auto kmain = hash_bucket(_pairs[slot].first); + if (kmain != bucket) + return kickout_bucket(kmain, bucket); + else if (next_bucket == bucket) + return _index[next_bucket].next = find_empty_bucket(next_bucket, 1); + + uint32_t csize = 1; + //find next linked bucket and check key + while (true) { + const auto eslot = _index[next_bucket].slot & _mask; + if (EMH_EQHASH(next_bucket, key_hash)) { + if (EMH_LIKELY(_eq(key, _pairs[eslot].first))) + return next_bucket; + } + + csize += 1; + const auto nbucket = _index[next_bucket].next; + if (nbucket == next_bucket) + break; + next_bucket = nbucket; + } + + //find a empty and link it to tail + const auto new_bucket = find_empty_bucket(next_bucket, csize); + prefetch_heap_block((char*)&_pairs[new_bucket]); + return _index[next_bucket].next = new_bucket; + } + + size_type find_unique_bucket(uint64_t key_hash) noexcept + { + const auto bucket = size_type(key_hash & _mask); + auto next_bucket = _index[bucket].next; + if ((int)next_bucket < 0) { +#if EMH_HIGH_LOAD + if (next_bucket != INACTIVE) + pop_empty(bucket); +#endif + return bucket; + } + + //check current bucket_key is in main bucket or not + const auto kmain = hash_main(bucket); + if (EMH_UNLIKELY(kmain != bucket)) + return kickout_bucket(kmain, bucket); + else if (EMH_UNLIKELY(next_bucket != bucket)) + next_bucket = find_last_bucket(next_bucket); + + return _index[next_bucket].next = find_empty_bucket(next_bucket, 2); + } + + /*** + Different probing techniques usually provide a trade-off between memory locality and avoidance of clustering. + Since Robin Hood hashing is relatively resilient to clustering (both primary and secondary), linear probing is the most cache friendly alternativeis typically used. + + It's the core algorithm of this hash map with highly optimization/benchmark. + normally linear probing is inefficient with high load factor, it use a new 3-way linear + probing strategy to search empty slot. from benchmark even the load factor > 0.9, it's more 2-3 timer fast than + one-way search strategy. + + 1. linear or quadratic probing a few cache line for less cache miss from input slot "bucket_from". + 2. the first search slot from member variant "_last", init with 0 + 3. the second search slot from calculated pos "(_num_filled + _last) & _mask", it's like a rand value + */ + // key is not in this mavalue. Find a place to put it. + size_type find_empty_bucket(const size_type bucket_from, uint32_t csize) noexcept + { + (void)csize; +#if EMH_HIGH_LOAD + if (_ehead) + return pop_empty(_ehead); +#endif + + auto bucket = bucket_from; + if (EMH_EMPTY(++bucket) || EMH_EMPTY(++bucket)) + return bucket; + +#ifdef EMH_QUADRATIC + constexpr size_type linear_probe_length = 2 * EMH_CACHE_LINE_SIZE / sizeof(Index);//16 + for (size_type offset = csize + 2, step = 4; offset <= linear_probe_length; ) { + bucket = (bucket_from + offset) & _mask; + if (EMH_EMPTY(bucket) || EMH_EMPTY(++bucket)) + return bucket; + offset += step; //7/8. 12. 16 + } +#else + constexpr size_type quadratic_probe_length = 6u; + for (size_type offset = 4u, step = 3u; step < quadratic_probe_length; ) { + bucket = (bucket_from + offset) & _mask; + if (EMH_EMPTY(bucket) || EMH_EMPTY(++bucket)) + return bucket; + offset += step++; + } +#endif + +#if EMH_PREFETCH + __builtin_prefetch(static_cast(_index + _last + 1), 0, EMH_PREFETCH); +#endif + + for (;;) { +#if EMH_PACK_TAIL + //find empty bucket and skip next + if (EMH_EMPTY(_last++))// || EMH_EMPTY(_last++)) + return _last++ - 1; + + if (EMH_UNLIKELY(_last >= _num_buckets)) + _last = 0; + + auto medium = (_mask / 4 + _last++) & _mask; + if (EMH_EMPTY(medium)) + return medium; +#else + _last &= _mask; + if (EMH_EMPTY(++_last))// || EMH_EMPTY(++_last)) + return _last; + + auto medium = (_num_buckets / 2 + _last) & _mask; + if (EMH_EMPTY(medium))// || EMH_EMPTY(++medium)) + return medium; +#endif + } + + return 0; + } + + size_type find_last_bucket(size_type main_bucket) const + { + auto next_bucket = _index[main_bucket].next; + if (next_bucket == main_bucket) + return main_bucket; + + while (true) { + const auto nbucket = _index[next_bucket].next; + if (nbucket == next_bucket) + return next_bucket; + next_bucket = nbucket; + } + } + + size_type find_prev_bucket(const size_type main_bucket, const size_type bucket) const + { + auto next_bucket = _index[main_bucket].next; + if (next_bucket == bucket) + return main_bucket; + + while (true) { + const auto nbucket = _index[next_bucket].next; + if (nbucket == bucket) + return next_bucket; + next_bucket = nbucket; + } + } + + size_type hash_bucket(const KeyT& key) const noexcept + { + return (size_type)hash_key(key) & _mask; + } + + size_type hash_main(const size_type bucket) const noexcept + { + const auto slot = _index[bucket].slot & _mask; + return (size_type)hash_key(_pairs[slot].first) & _mask; + } + +#if EMH_INT_HASH + static constexpr uint64_t KC = UINT64_C(11400714819323198485); + static uint64_t hash64(uint64_t key) + { +#if __SIZEOF_INT128__ && EMH_INT_HASH == 1 + __uint128_t r = key; r *= KC; + return (uint64_t)(r >> 64) + (uint64_t)r; +#elif EMH_INT_HASH == 2 + //MurmurHash3Mixer + uint64_t h = key; + h ^= h >> 33; + h *= 0xff51afd7ed558ccd; + h ^= h >> 33; + h *= 0xc4ceb9fe1a85ec53; + h ^= h >> 33; + return h; +#elif _WIN64 && EMH_INT_HASH == 1 + uint64_t high; + return _umul128(key, KC, &high) + high; +#elif EMH_INT_HASH == 3 + auto ror = (key >> 32) | (key << 32); + auto low = key * 0xA24BAED4963EE407ull; + auto high = ror * 0x9FB21C651E98DF25ull; + auto mix = low + high; + return mix; +#elif EMH_INT_HASH == 1 + uint64_t r = key * UINT64_C(0xca4bcaa75ec3f625); + return (r >> 32) + r; +#elif EMH_WYHASH64 + return wyhash64(key, KC); +#else + uint64_t x = key; + x = (x ^ (x >> 30)) * UINT64_C(0xbf58476d1ce4e5b9); + x = (x ^ (x >> 27)) * UINT64_C(0x94d049bb133111eb); + x = x ^ (x >> 31); + return x; +#endif + } +#endif + +#if EMH_WYHASH_HASH + //#define WYHASH_CONDOM 1 + static uint64_t wymix(uint64_t A, uint64_t B) + { +#if defined(__SIZEOF_INT128__) + __uint128_t r = A; r *= B; +#if WYHASH_CONDOM2 + A ^= (uint64_t)r; B ^= (uint64_t)(r >> 64); +#else + A = (uint64_t)r; B = (uint64_t)(r >> 64); +#endif + +#elif defined(_MSC_VER) && defined(_M_X64) +#if WYHASH_CONDOM2 + uint64_t a, b; + a = _umul128(A, B, &b); + A ^= a; B ^= b; +#else + A = _umul128(A, B, &B); +#endif +#else + uint64_t ha = A >> 32, hb = B >> 32, la = (uint32_t)A, lb = (uint32_t)B, hi, lo; + uint64_t rh = ha * hb, rm0 = ha * lb, rm1 = hb * la, rl = la * lb, t = rl + (rm0 << 32), c = t < rl; + lo = t + (rm1 << 32); c += lo < t; hi = rh + (rm0 >> 32) + (rm1 >> 32) + c; +#if WYHASH_CONDOM2 + A ^= lo; B ^= hi; +#else + A = lo; B = hi; +#endif +#endif + return A ^ B; + } + + //multiply and xor mix function, aka MUM + static inline uint64_t wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return v; } + static inline uint64_t wyr4(const uint8_t *p) { uint32_t v; memcpy(&v, p, 4); return v; } + static inline uint64_t wyr3(const uint8_t *p, size_t k) { + return (((uint64_t)p[0]) << 16) | (((uint64_t)p[k >> 1]) << 8) | p[k - 1]; + } + + inline static const uint64_t secret[4] = { + 0x2d358dccaa6c78a5ull, 0x8bb84b93962eacc9ull, + 0x4b33a62ed433d4a3ull, 0x4d5a2da51de1aa47ull}; +public: + //wyhash main function https://github.com/wangyi-fudan/wyhash + static uint64_t wyhashstr(const char *key, const size_t len) + { + uint64_t a = 0, b = 0, seed = secret[0]; + const uint8_t *p = (const uint8_t*)key; + if (EMH_LIKELY(len <= 16)) { + if (EMH_LIKELY(len >= 4)) { + const auto half = (len >> 3) << 2; + a = (wyr4(p) << 32U) | wyr4(p + half); p += len - 4; + b = (wyr4(p) << 32U) | wyr4(p - half); + } else if (len) { + a = wyr3(p, len); + } + } else { + size_t i = len; + if (EMH_UNLIKELY(i > 48)) { + uint64_t see1 = seed, see2 = seed; + do { + seed = wymix(wyr8(p + 0) ^ secret[1], wyr8(p + 8) ^ seed); + see1 = wymix(wyr8(p + 16) ^ secret[2], wyr8(p + 24) ^ see1); + see2 = wymix(wyr8(p + 32) ^ secret[3], wyr8(p + 40) ^ see2); + p += 48; i -= 48; + } while (EMH_LIKELY(i > 48)); + seed ^= see1 ^ see2; + } + while (i > 16) { + seed = wymix(wyr8(p) ^ secret[1], wyr8(p + 8) ^ seed); + i -= 16; p += 16; + } + a = wyr8(p + i - 16); + b = wyr8(p + i - 8); + } + + return wymix(secret[1] ^ len, wymix(a ^ secret[1], b ^ seed)); + } +#endif + +private: + template::value, uint32_t>::type = 0> + inline uint64_t hash_key(const UType key) const + { +#if EMH_INT_HASH + return hash64(key); +#elif EMH_IDENTITY_HASH + return key + (key >> 24); +#else + return _hasher(key); +#endif + } + + template::value, uint32_t>::type = 0> + inline uint64_t hash_key(const UType& key) const + { +#if EMH_WYHASH_HASH + return wyhashstr(key.data(), key.size()); +#else + return _hasher(key); +#endif + } + + template::value && !std::is_same::value, uint32_t>::type = 0> + inline uint64_t hash_key(const UType& key) const + { + return _hasher(key); + } + +private: + Index* _index; + value_type*_pairs; + + HashT _hasher; + EqT _eq; + uint32_t _mlf; + size_type _mask; + size_type _num_buckets; + size_type _num_filled; + size_type _last; +#if EMH_HIGH_LOAD + size_type _ehead; +#endif + size_type _etail; +}; +} // namespace emhash + diff --git a/src/third_party/rapidhash/README.ninja b/src/third_party/rapidhash/README.ninja new file mode 100644 index 0000000000..1d74b67c1f --- /dev/null +++ b/src/third_party/rapidhash/README.ninja @@ -0,0 +1,7 @@ +Description: Very fast, high quality, platform-independent hashing algorithm. +Version: commit 4a6b2570e868536be84800353efd92c699f37d2c +URL: https://github.com/Nicoshev/rapidhash +Copyright: Copyright (C) 2024 Nicolas De Carli, Based on 'wyhash', by Wang Yi +SPDX-License-Identifier: BSD-2-Clause +Local changes: + - Changed to UNIX line endings diff --git a/src/third_party/rapidhash/rapidhash.h b/src/third_party/rapidhash/rapidhash.h new file mode 100755 index 0000000000..463f733d85 --- /dev/null +++ b/src/third_party/rapidhash/rapidhash.h @@ -0,0 +1,323 @@ +/* + * rapidhash - Very fast, high quality, platform-independent hashing algorithm. + * Copyright (C) 2024 Nicolas De Carli + * + * Based on 'wyhash', by Wang Yi + * + * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php) + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * You can contact the author at: + * - rapidhash source repository: https://github.com/Nicoshev/rapidhash + */ + +/* + * Includes. + */ +#include +#include +#if defined(_MSC_VER) + #include + #if defined(_M_X64) && !defined(_M_ARM64EC) + #pragma intrinsic(_umul128) + #endif +#endif + +/* + * C++ macros. + * + * RAPIDHASH_INLINE can be overridden to be stronger than a hint, i.e. by adding __attribute__((always_inline)). + */ +#ifdef __cplusplus + #define RAPIDHASH_NOEXCEPT noexcept + #define RAPIDHASH_CONSTEXPR constexpr + #ifndef RAPIDHASH_INLINE + #define RAPIDHASH_INLINE inline + #endif +#else + #define RAPIDHASH_NOEXCEPT + #define RAPIDHASH_CONSTEXPR static const + #ifndef RAPIDHASH_INLINE + #define RAPIDHASH_INLINE static inline + #endif +#endif + +/* + * Protection macro, alters behaviour of rapid_mum multiplication function. + * + * RAPIDHASH_FAST: Normal behavior, max speed. + * RAPIDHASH_PROTECTED: Extra protection against entropy loss. + */ +#ifndef RAPIDHASH_PROTECTED + #define RAPIDHASH_FAST +#elif defined(RAPIDHASH_FAST) + #error "cannot define RAPIDHASH_PROTECTED and RAPIDHASH_FAST simultaneously." +#endif + +/* + * Unrolling macros, changes code definition for main hash function. + * + * RAPIDHASH_COMPACT: Legacy variant, each loop process 48 bytes. + * RAPIDHASH_UNROLLED: Unrolled variant, each loop process 96 bytes. + * + * Most modern CPUs should benefit from having RAPIDHASH_UNROLLED. + * + * These macros do not alter the output hash. + */ +#ifndef RAPIDHASH_COMPACT + #define RAPIDHASH_UNROLLED +#elif defined(RAPIDHASH_UNROLLED) + #error "cannot define RAPIDHASH_COMPACT and RAPIDHASH_UNROLLED simultaneously." +#endif + +/* + * Likely and unlikely macros. + */ +#if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) + #define _likely_(x) __builtin_expect(x,1) + #define _unlikely_(x) __builtin_expect(x,0) +#else + #define _likely_(x) (x) + #define _unlikely_(x) (x) +#endif + +/* + * Endianness macros. + */ +#ifndef RAPIDHASH_LITTLE_ENDIAN + #if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) + #define RAPIDHASH_LITTLE_ENDIAN + #elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) + #define RAPIDHASH_BIG_ENDIAN + #else + #warning "could not determine endianness! Falling back to little endian." + #define RAPIDHASH_LITTLE_ENDIAN + #endif +#endif + +/* + * Default seed. + */ +#define RAPID_SEED (0xbdd89aa982704029ull) + +/* + * Default secret parameters. + */ +RAPIDHASH_CONSTEXPR uint64_t rapid_secret[3] = {0x2d358dccaa6c78a5ull, 0x8bb84b93962eacc9ull, 0x4b33a62ed433d4a3ull}; + +/* + * 64*64 -> 128bit multiply function. + * + * @param A Address of 64-bit number. + * @param B Address of 64-bit number. + * + * Calculates 128-bit C = *A * *B. + * + * When RAPIDHASH_FAST is defined: + * Overwrites A contents with C's low 64 bits. + * Overwrites B contents with C's high 64 bits. + * + * When RAPIDHASH_PROTECTED is defined: + * Xors and overwrites A contents with C's low 64 bits. + * Xors and overwrites B contents with C's high 64 bits. + */ +RAPIDHASH_INLINE void rapid_mum(uint64_t *A, uint64_t *B) RAPIDHASH_NOEXCEPT { +#if defined(__SIZEOF_INT128__) + __uint128_t r=*A; r*=*B; + #ifdef RAPIDHASH_PROTECTED + *A^=(uint64_t)r; *B^=(uint64_t)(r>>64); + #else + *A=(uint64_t)r; *B=(uint64_t)(r>>64); + #endif +#elif defined(_MSC_VER) && (defined(_WIN64) || defined(_M_HYBRID_CHPE_ARM64)) + #if defined(_M_X64) + #ifdef RAPIDHASH_PROTECTED + uint64_t a, b; + a=_umul128(*A,*B,&b); + *A^=a; *B^=b; + #else + *A=_umul128(*A,*B,B); + #endif + #else + #ifdef RAPIDHASH_PROTECTED + uint64_t a, b; + b = __umulh(*A, *B); + a = *A * *B; + *A^=a; *B^=b; + #else + uint64_t c = __umulh(*A, *B); + *A = *A * *B; + *B = c; + #endif + #endif +#else + uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B, hi, lo; + uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t>32)+(rm1>>32)+c; + #ifdef RAPIDHASH_PROTECTED + *A^=lo; *B^=hi; + #else + *A=lo; *B=hi; + #endif +#endif +} + +/* + * Multiply and xor mix function. + * + * @param A 64-bit number. + * @param B 64-bit number. + * + * Calculates 128-bit C = A * B. + * Returns 64-bit xor between high and low 64 bits of C. + */ +RAPIDHASH_INLINE uint64_t rapid_mix(uint64_t A, uint64_t B) RAPIDHASH_NOEXCEPT { rapid_mum(&A,&B); return A^B; } + +/* + * Read functions. + */ +#ifdef RAPIDHASH_LITTLE_ENDIAN +RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return v;} +RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return v;} +#elif defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__) +RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return __builtin_bswap64(v);} +RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return __builtin_bswap32(v);} +#elif defined(_MSC_VER) +RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint64_t v; memcpy(&v, p, sizeof(uint64_t)); return _byteswap_uint64(v);} +RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { uint32_t v; memcpy(&v, p, sizeof(uint32_t)); return _byteswap_ulong(v);} +#else +RAPIDHASH_INLINE uint64_t rapid_read64(const uint8_t *p) RAPIDHASH_NOEXCEPT { + uint64_t v; memcpy(&v, p, 8); + return (((v >> 56) & 0xff)| ((v >> 40) & 0xff00)| ((v >> 24) & 0xff0000)| ((v >> 8) & 0xff000000)| ((v << 8) & 0xff00000000)| ((v << 24) & 0xff0000000000)| ((v << 40) & 0xff000000000000)| ((v << 56) & 0xff00000000000000)); +} +RAPIDHASH_INLINE uint64_t rapid_read32(const uint8_t *p) RAPIDHASH_NOEXCEPT { + uint32_t v; memcpy(&v, p, 4); + return (((v >> 24) & 0xff)| ((v >> 8) & 0xff00)| ((v << 8) & 0xff0000)| ((v << 24) & 0xff000000)); +} +#endif + +/* + * Reads and combines 3 bytes of input. + * + * @param p Buffer to read from. + * @param k Length of @p, in bytes. + * + * Always reads and combines 3 bytes from memory. + * Guarantees to read each buffer position at least once. + * + * Returns a 64-bit value containing all three bytes read. + */ +RAPIDHASH_INLINE uint64_t rapid_readSmall(const uint8_t *p, size_t k) RAPIDHASH_NOEXCEPT { return (((uint64_t)p[0])<<56)|(((uint64_t)p[k>>1])<<32)|p[k-1];} + +/* + * rapidhash main function. + * + * @param key Buffer to be hashed. + * @param len @key length, in bytes. + * @param seed 64-bit seed used to alter the hash result predictably. + * @param secret Triplet of 64-bit secrets used to alter hash result predictably. + * + * Returns a 64-bit hash. + */ +RAPIDHASH_INLINE uint64_t rapidhash_internal(const void *key, size_t len, uint64_t seed, const uint64_t* secret) RAPIDHASH_NOEXCEPT { + const uint8_t *p=(const uint8_t *)key; seed^=rapid_mix(seed^secret[0],secret[1])^len; uint64_t a, b; + if(_likely_(len<=16)){ + if(_likely_(len>=4)){ + const uint8_t * plast = p + len - 4; + a = (rapid_read32(p) << 32) | rapid_read32(plast); + const uint64_t delta = ((len&24)>>(len>>3)); + b = ((rapid_read32(p + delta) << 32) | rapid_read32(plast - delta)); } + else if(_likely_(len>0)){ a=rapid_readSmall(p,len); b=0;} + else a=b=0; + } + else{ + size_t i=len; + if(_unlikely_(i>48)){ + uint64_t see1=seed, see2=seed; +#ifdef RAPIDHASH_UNROLLED + while(_likely_(i>=96)){ + seed=rapid_mix(rapid_read64(p)^secret[0],rapid_read64(p+8)^seed); + see1=rapid_mix(rapid_read64(p+16)^secret[1],rapid_read64(p+24)^see1); + see2=rapid_mix(rapid_read64(p+32)^secret[2],rapid_read64(p+40)^see2); + seed=rapid_mix(rapid_read64(p+48)^secret[0],rapid_read64(p+56)^seed); + see1=rapid_mix(rapid_read64(p+64)^secret[1],rapid_read64(p+72)^see1); + see2=rapid_mix(rapid_read64(p+80)^secret[2],rapid_read64(p+88)^see2); + p+=96; i-=96; + } + if(_unlikely_(i>=48)){ + seed=rapid_mix(rapid_read64(p)^secret[0],rapid_read64(p+8)^seed); + see1=rapid_mix(rapid_read64(p+16)^secret[1],rapid_read64(p+24)^see1); + see2=rapid_mix(rapid_read64(p+32)^secret[2],rapid_read64(p+40)^see2); + p+=48; i-=48; + } +#else + do { + seed=rapid_mix(rapid_read64(p)^secret[0],rapid_read64(p+8)^seed); + see1=rapid_mix(rapid_read64(p+16)^secret[1],rapid_read64(p+24)^see1); + see2=rapid_mix(rapid_read64(p+32)^secret[2],rapid_read64(p+40)^see2); + p+=48; i-=48; + } while (_likely_(i>=48)); +#endif + seed^=see1^see2; + } + if(i>16){ + seed=rapid_mix(rapid_read64(p)^secret[2],rapid_read64(p+8)^seed^secret[1]); + if(i>32) + seed=rapid_mix(rapid_read64(p+16)^secret[2],rapid_read64(p+24)^seed); + } + a=rapid_read64(p+i-16); b=rapid_read64(p+i-8); + } + a^=secret[1]; b^=seed; rapid_mum(&a,&b); + return rapid_mix(a^secret[0]^len,b^secret[1]); +} + +/* + * rapidhash default seeded hash function. + * + * @param key Buffer to be hashed. + * @param len @key length, in bytes. + * @param seed 64-bit seed used to alter the hash result predictably. + * + * Calls rapidhash_internal using provided parameters and default secrets. + * + * Returns a 64-bit hash. + */ +RAPIDHASH_INLINE uint64_t rapidhash_withSeed(const void *key, size_t len, uint64_t seed) RAPIDHASH_NOEXCEPT { + return rapidhash_internal(key, len, seed, rapid_secret); +} + +/* + * rapidhash default hash function. + * + * @param key Buffer to be hashed. + * @param len @key length, in bytes. + * + * Calls rapidhash_withSeed using provided parameters and the default seed. + * + * Returns a 64-bit hash. + */ +RAPIDHASH_INLINE uint64_t rapidhash(const void *key, size_t len) RAPIDHASH_NOEXCEPT { + return rapidhash_withSeed(key, len, RAPID_SEED); +} diff --git a/src/util.cc b/src/util.cc index ee810d6e47..7000a6ad12 100644 --- a/src/util.cc +++ b/src/util.cc @@ -21,6 +21,7 @@ #include #include #include +#include #endif #include @@ -45,14 +46,22 @@ #elif defined(__SVR4) && defined(__sun) #include #include -#elif defined(_AIX) +#elif defined(_AIX) && !defined(__PASE__) #include -#elif defined(linux) || defined(__GLIBC__) +#elif defined(__linux__) || defined(__GLIBC__) #include +#include +#include +#include "string_piece_util.h" +#endif + +#if defined(__FreeBSD__) +#include #endif #include "edit_distance.h" -#include "metrics.h" + +using namespace std; void Fatal(const char* msg, ...) { va_list ap; @@ -72,34 +81,52 @@ void Fatal(const char* msg, ...) { #endif } +void Warning(const char* msg, va_list ap) { + fprintf(stderr, "ninja: warning: "); + vfprintf(stderr, msg, ap); + fprintf(stderr, "\n"); +} + void Warning(const char* msg, ...) { va_list ap; - fprintf(stderr, "ninja: warning: "); va_start(ap, msg); - vfprintf(stderr, msg, ap); + Warning(msg, ap); va_end(ap); +} + +void Error(const char* msg, va_list ap) { + fprintf(stderr, "ninja: error: "); + vfprintf(stderr, msg, ap); fprintf(stderr, "\n"); } void Error(const char* msg, ...) { va_list ap; - fprintf(stderr, "ninja: error: "); va_start(ap, msg); - vfprintf(stderr, msg, ap); + Error(msg, ap); + va_end(ap); +} + +void Info(const char* msg, va_list ap) { + fprintf(stdout, "ninja: "); + vfprintf(stdout, msg, ap); + fprintf(stdout, "\n"); +} + +void Info(const char* msg, ...) { + va_list ap; + va_start(ap, msg); + Info(msg, ap); va_end(ap); - fprintf(stderr, "\n"); } -bool CanonicalizePath(string* path, uint64_t* slash_bits, string* err) { - METRIC_RECORD("canonicalize str"); +void CanonicalizePath(string* path, uint64_t* slash_bits) { size_t len = path->size(); char* str = 0; if (len > 0) str = &(*path)[0]; - if (!CanonicalizePath(str, &len, slash_bits, err)) - return false; + CanonicalizePath(str, &len, slash_bits); path->resize(len); - return true; } static bool IsPathSeparator(char c) { @@ -110,30 +137,26 @@ static bool IsPathSeparator(char c) { #endif } -bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, - string* err) { +void CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits) { // WARNING: this function is performance-critical; please benchmark // any changes you make to it. - METRIC_RECORD("canonicalize path"); if (*len == 0) { - *err = "empty path"; - return false; + return; } - const int kMaxPathComponents = 60; - char* components[kMaxPathComponents]; - int component_count = 0; - char* start = path; char* dst = start; + char* dst_start = dst; const char* src = start; const char* end = start + *len; + const char* src_next; + // For absolute paths, skip the leading directory separator + // as this one should never be removed from the result. if (IsPathSeparator(*src)) { #ifdef _WIN32 - - // network path starts with // - if (*len > 1 && IsPathSeparator(*(src + 1))) { + // Windows network path starts with // + if (src + 2 <= end && IsPathSeparator(src[1])) { src += 2; dst += 2; } else { @@ -144,50 +167,126 @@ bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, ++src; ++dst; #endif + dst_start = dst; + } else { + // For relative paths, skip any leading ../ as these are quite common + // to reference source files in build plans, and doing this here makes + // the loop work below faster in general. + while (src + 3 <= end && src[0] == '.' && src[1] == '.' && + IsPathSeparator(src[2])) { + src += 3; + dst += 3; + } } - while (src < end) { - if (*src == '.') { - if (src + 1 == end || IsPathSeparator(src[1])) { - // '.' component; eliminate. - src += 2; - continue; - } else if (src[1] == '.' && (src + 2 == end || IsPathSeparator(src[2]))) { - // '..' component. Back up if possible. + // Loop over all components of the paths _except_ the last one, in + // order to simplify the loop's code and make it faster. + int component_count = 0; + char* dst0 = dst; + for (; src < end; src = src_next) { +#ifndef _WIN32 + // Use memchr() for faster lookups thanks to optimized C library + // implementation. `hyperfine canon_perftest` shows a significant + // difference (e,g, 484ms vs 437ms). + const char* next_sep = + static_cast(::memchr(src, '/', end - src)); + if (!next_sep) { + // This is the last component, will be handled out of the loop. + break; + } +#else + // Need to check for both '/' and '\\' so do not use memchr(). + // Cannot use strpbrk() because end[0] can be \0 or something else! + const char* next_sep = src; + while (next_sep != end && !IsPathSeparator(*next_sep)) + ++next_sep; + if (next_sep == end) { + // This is the last component, will be handled out of the loop. + break; + } +#endif + // Position for next loop iteration. + src_next = next_sep + 1; + // Length of the component, excluding trailing directory. + size_t component_len = next_sep - src; + + if (component_len <= 2) { + if (component_len == 0) { + continue; // Ignore empty component, e.g. 'foo//bar' -> 'foo/bar'. + } + if (src[0] == '.') { + if (component_len == 1) { + continue; // Ignore '.' component, e.g. './foo' -> 'foo'. + } else if (src[1] == '.') { + // Process the '..' component if found. Back up if possible. + if (component_count > 0) { + // Move back to start of previous component. + --component_count; + while (--dst > dst0 && !IsPathSeparator(dst[-1])) { + // nothing to do here, decrement happens before condition check. + } + } else { + dst[0] = '.'; + dst[1] = '.'; + dst[2] = src[2]; + dst += 3; + } + continue; + } + } + } + ++component_count; + + // Copy or skip component, including trailing directory separator. + if (dst != src) { + ::memmove(dst, src, src_next - src); + } + dst += src_next - src; + } + + // Handling the last component that does not have a trailing separator. + // The logic here is _slightly_ different since there is no trailing + // directory separator. + size_t component_len = end - src; + do { + if (component_len == 0) + break; // Ignore empty component (e.g. 'foo//' -> 'foo/') + if (src[0] == '.') { + if (component_len == 1) + break; // Ignore trailing '.' (e.g. 'foo/.' -> 'foo/') + if (component_len == 2 && src[1] == '.') { + // Handle '..'. Back up if possible. if (component_count > 0) { - dst = components[component_count - 1]; - src += 3; - --component_count; + while (--dst > dst0 && !IsPathSeparator(dst[-1])) { + // nothing to do here, decrement happens before condition check. + } } else { - *dst++ = *src++; - *dst++ = *src++; - *dst++ = *src++; + dst[0] = '.'; + dst[1] = '.'; + dst += 2; + // No separator to add here. } - continue; + break; } } - - if (IsPathSeparator(*src)) { - src++; - continue; + // Skip or copy last component, no trailing separator. + if (dst != src) { + ::memmove(dst, src, component_len); } + dst += component_len; + } while (0); - if (component_count == kMaxPathComponents) - Fatal("path has too many components : %s", path); - components[component_count] = dst; - ++component_count; - - while (src != end && !IsPathSeparator(*src)) - *dst++ = *src++; - *dst++ = *src++; // Copy '/' or final \0 character as well. - } + // Remove trailing path separator if any, but keep the initial + // path separator(s) if there was one (or two on Windows). + if (dst > dst_start && IsPathSeparator(dst[-1])) + dst--; if (dst == start) { + // Handle special cases like "aa/.." -> "." *dst++ = '.'; - *dst++ = '\0'; } - *len = dst - start - 1; + *len = dst - start; // dst points after the trailing char here. #ifdef _WIN32 uint64_t bits = 0; uint64_t bits_mask = 1; @@ -207,7 +306,6 @@ bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, #else *slash_bits = 0; #endif - return true; } static inline bool IsKnownShellSafeCharacter(char ch) { @@ -331,7 +429,8 @@ int ReadFile(const string& path, string* contents, string* err) { if (!::ReadFile(f, buf, sizeof(buf), &len, NULL)) { err->assign(GetLastErrorString()); contents->clear(); - return -1; + ::CloseHandle(f); + return -EIO; } if (len == 0) break; @@ -346,8 +445,13 @@ int ReadFile(const string& path, string* contents, string* err) { return -errno; } +#ifdef __USE_LARGEFILE64 + struct stat64 st; + if (fstat64(fileno(f), &st) < 0) { +#else struct stat st; if (fstat(fileno(f), &st) < 0) { +#endif err->assign(strerror(errno)); fclose(f); return -errno; @@ -433,10 +537,17 @@ string GetLastErrorString() { FORMAT_MESSAGE_IGNORE_INSERTS, NULL, err, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), + MAKELANGID(LANG_ENGLISH, SUBLANG_DEFAULT), (char*)&msg_buf, 0, NULL); + + if (msg_buf == nullptr) { + char fallback_msg[128] = {0}; + snprintf(fallback_msg, sizeof(fallback_msg), "GetLastError() = %lu", err); + return fallback_msg; + } + string msg = msg_buf; LocalFree(msg_buf); return msg; @@ -479,22 +590,228 @@ string StripAnsiEscapeCodes(const string& in) { return stripped; } +#if defined(__linux__) || defined(__GLIBC__) +std::pair readCount(const std::string& path) { + std::ifstream file(path.c_str()); + if (!file.is_open()) + return std::make_pair(0, false); + int64_t n = 0; + file >> n; + if (file.good()) + return std::make_pair(n, true); + return std::make_pair(0, false); +} + +struct MountPoint { + int mountId; + int parentId; + StringPiece deviceId; + StringPiece root; + StringPiece mountPoint; + vector options; + vector optionalFields; + StringPiece fsType; + StringPiece mountSource; + vector superOptions; + bool parse(const string& line) { + vector pieces = SplitStringPiece(line, ' '); + if (pieces.size() < 10) + return false; + size_t optionalStart = 0; + for (size_t i = 6; i < pieces.size(); i++) { + if (pieces[i] == "-") { + optionalStart = i + 1; + break; + } + } + if (optionalStart == 0) + return false; + if (optionalStart + 3 != pieces.size()) + return false; + mountId = atoi(pieces[0].AsString().c_str()); + parentId = atoi(pieces[1].AsString().c_str()); + deviceId = pieces[2]; + root = pieces[3]; + mountPoint = pieces[4]; + options = SplitStringPiece(pieces[5], ','); + optionalFields = + vector(&pieces[6], &pieces[optionalStart - 1]); + fsType = pieces[optionalStart]; + mountSource = pieces[optionalStart + 1]; + superOptions = SplitStringPiece(pieces[optionalStart + 2], ','); + return true; + } + string translate(string& path) const { + // path must be sub dir of root + if (path.compare(0, root.len_, root.str_, root.len_) != 0) { + return string(); + } + path.erase(0, root.len_); + if (path == ".." || (path.length() > 2 && path.compare(0, 3, "../") == 0)) { + return string(); + } + return mountPoint.AsString() + "/" + path; + } +}; + +struct CGroupSubSys { + int id; + string name; + vector subsystems; + bool parse(string& line) { + size_t first = line.find(':'); + if (first == string::npos) + return false; + line[first] = '\0'; + size_t second = line.find(':', first + 1); + if (second == string::npos) + return false; + line[second] = '\0'; + id = atoi(line.c_str()); + name = line.substr(second + 1); + vector pieces = + SplitStringPiece(StringPiece(line.c_str() + first + 1), ','); + for (size_t i = 0; i < pieces.size(); i++) { + subsystems.push_back(pieces[i].AsString()); + } + return true; + } +}; + +map ParseMountInfo(map& subsystems) { + map cgroups; + ifstream mountinfo("/proc/self/mountinfo"); + if (!mountinfo.is_open()) + return cgroups; + while (!mountinfo.eof()) { + string line; + getline(mountinfo, line); + MountPoint mp; + if (!mp.parse(line)) + continue; + if (mp.fsType != "cgroup") + continue; + for (size_t i = 0; i < mp.superOptions.size(); i++) { + string opt = mp.superOptions[i].AsString(); + map::iterator subsys = subsystems.find(opt); + if (subsys == subsystems.end()) + continue; + string newPath = mp.translate(subsys->second.name); + if (!newPath.empty()) + cgroups.insert(make_pair(opt, newPath)); + } + } + return cgroups; +} + +map ParseSelfCGroup() { + map cgroups; + ifstream cgroup("/proc/self/cgroup"); + if (!cgroup.is_open()) + return cgroups; + string line; + while (!cgroup.eof()) { + getline(cgroup, line); + CGroupSubSys subsys; + if (!subsys.parse(line)) + continue; + for (size_t i = 0; i < subsys.subsystems.size(); i++) { + cgroups.insert(make_pair(subsys.subsystems[i], subsys)); + } + } + return cgroups; +} + +int ParseCPUFromCGroup() { + map subsystems = ParseSelfCGroup(); + map cgroups = ParseMountInfo(subsystems); + map::iterator cpu = cgroups.find("cpu"); + if (cpu == cgroups.end()) + return -1; + std::pair quota = readCount(cpu->second + "/cpu.cfs_quota_us"); + if (!quota.second || quota.first == -1) + return -1; + std::pair period = + readCount(cpu->second + "/cpu.cfs_period_us"); + if (!period.second) + return -1; + if (period.first == 0) + return -1; + return quota.first / period.first; +} +#endif + int GetProcessorCount() { #ifdef _WIN32 - SYSTEM_INFO info; - GetNativeSystemInfo(&info); - return info.dwNumberOfProcessors; + DWORD cpuCount = 0; +#ifndef _WIN64 + // Need to use GetLogicalProcessorInformationEx to get real core count on + // machines with >64 cores. See https://stackoverflow.com/a/31209344/21475 + DWORD len = 0; + if (!GetLogicalProcessorInformationEx(RelationProcessorCore, nullptr, &len) + && GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + std::vector buf(len); + int cores = 0; + if (GetLogicalProcessorInformationEx(RelationProcessorCore, + reinterpret_cast( + buf.data()), &len)) { + for (DWORD i = 0; i < len; ) { + auto info = reinterpret_cast( + buf.data() + i); + if (info->Relationship == RelationProcessorCore && + info->Processor.GroupCount == 1) { + for (KAFFINITY core_mask = info->Processor.GroupMask[0].Mask; + core_mask; core_mask >>= 1) { + cores += (core_mask & 1); + } + } + i += info->Size; + } + if (cores != 0) { + cpuCount = cores; + } + } + } +#endif + if (cpuCount == 0) { + cpuCount = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); + } + JOBOBJECT_CPU_RATE_CONTROL_INFORMATION info; + // reference: + // https://docs.microsoft.com/en-us/windows/win32/api/winnt/ns-winnt-jobobject_cpu_rate_control_information + if (QueryInformationJobObject(NULL, JobObjectCpuRateControlInformation, &info, + sizeof(info), NULL)) { + if (info.ControlFlags & (JOB_OBJECT_CPU_RATE_CONTROL_ENABLE | + JOB_OBJECT_CPU_RATE_CONTROL_HARD_CAP)) { + return cpuCount * info.CpuRate / 10000; + } + } + return cpuCount; #else -#ifdef CPU_COUNT + int cgroupCount = -1; + int schedCount = -1; +#if defined(__linux__) || defined(__GLIBC__) + cgroupCount = ParseCPUFromCGroup(); +#endif // The number of exposed processors might not represent the actual number of // processors threads can run on. This happens when a CPU set limitation is // active, see https://github.com/ninja-build/ninja/issues/1278 +#if defined(__FreeBSD__) + cpuset_t mask; + CPU_ZERO(&mask); + if (cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID, -1, sizeof(mask), + &mask) == 0) { + return CPU_COUNT(&mask); + } +#elif defined(CPU_COUNT) cpu_set_t set; if (sched_getaffinity(getpid(), sizeof(set), &set) == 0) { - return CPU_COUNT(&set); + schedCount = CPU_COUNT(&set); } #endif - return sysconf(_SC_NPROCESSORS_ONLN); + if (cgroupCount >= 0 && schedCount >= 0) return std::min(cgroupCount, schedCount); + if (cgroupCount < 0 && schedCount < 0) return sysconf(_SC_NPROCESSORS_ONLN); + return std::max(cgroupCount, schedCount); #endif } @@ -564,6 +881,10 @@ double GetLoadAverage() { return posix_compatible_load; } +#elif defined(__PASE__) +double GetLoadAverage() { + return -0.0f; +} #elif defined(_AIX) double GetLoadAverage() { perfstat_cpu_total_t cpu_stats; @@ -574,13 +895,17 @@ double GetLoadAverage() { // Calculation taken from comment in libperfstats.h return double(cpu_stats.loadavg[0]) / double(1 << SBITS); } -#elif defined(__UCLIBC__) +#elif defined(__UCLIBC__) || (defined(__BIONIC__) && __ANDROID_API__ < 29) double GetLoadAverage() { struct sysinfo si; if (sysinfo(&si) != 0) return -0.0f; return 1.0 / (1 << SI_LOAD_SHIFT) * si.loads[0]; } +#elif defined(__HAIKU__) +double GetLoadAverage() { + return -0.0f; +} #else double GetLoadAverage() { double loadavg[3] = { 0.0f, 0.0f, 0.0f }; @@ -593,16 +918,19 @@ double GetLoadAverage() { } #endif // _WIN32 -string ElideMiddle(const string& str, size_t width) { - const int kMargin = 3; // Space for "...". - string result = str; - if (result.size() > width) { - size_t elide_size = (width - kMargin) / 2; - result = result.substr(0, elide_size) - + "..." - + result.substr(result.size() - elide_size, elide_size); +std::string GetWorkingDirectory() { + std::string ret; + char* success = NULL; + do { + ret.resize(ret.size() + 1024); + errno = 0; + success = getcwd(&ret[0], ret.size()); + } while (!success && errno == ERANGE); + if (!success) { + Fatal("cannot determine working directory: %s", strerror(errno)); } - return result; + ret.resize(strlen(&ret[0])); + return ret; } bool Truncate(const string& path, size_t size, string* err) { @@ -622,3 +950,11 @@ bool Truncate(const string& path, size_t size, string* err) { } return true; } + +int platformAwareUnlink(const char* filename) { + #ifdef _WIN32 + return _unlink(filename); + #else + return unlink(filename); + #endif +} diff --git a/src/util.h b/src/util.h index 6a4a7a9f84..02c2418396 100644 --- a/src/util.h +++ b/src/util.h @@ -21,65 +21,70 @@ #include #endif +#include + #include #include -using namespace std; -#ifdef _MSC_VER -#define NORETURN __declspec(noreturn) +#if !defined(__has_cpp_attribute) +# define __has_cpp_attribute(x) 0 +#endif + +#if __has_cpp_attribute(noreturn) +# define NORETURN [[noreturn]] #else -#define NORETURN __attribute__((noreturn)) +# define NORETURN // nothing for old compilers #endif /// Log a fatal message and exit. NORETURN void Fatal(const char* msg, ...); // Have a generic fall-through for different versions of C/C++. -#if defined(__cplusplus) && __cplusplus >= 201703L -#define NINJA_FALLTHROUGH [[fallthrough]] -#elif defined(__cplusplus) && __cplusplus >= 201103L && defined(__clang__) -#define NINJA_FALLTHROUGH [[clang::fallthrough]] -#elif defined(__cplusplus) && __cplusplus >= 201103L && defined(__GNUC__) && \ - __GNUC__ >= 7 -#define NINJA_FALLTHROUGH [[gnu::fallthrough]] -#elif defined(__GNUC__) && __GNUC__ >= 7 // gcc 7 -#define NINJA_FALLTHROUGH __attribute__ ((fallthrough)) -#else // C++11 on gcc 6, and all other cases -#define NINJA_FALLTHROUGH +#if __has_cpp_attribute(fallthrough) +# define NINJA_FALLTHROUGH [[fallthrough]] +#elif defined(__clang__) +# define NINJA_FALLTHROUGH [[clang::fallthrough]] +#else +# define NINJA_FALLTHROUGH // nothing #endif /// Log a warning message. void Warning(const char* msg, ...); +void Warning(const char* msg, va_list ap); /// Log an error message. void Error(const char* msg, ...); +void Error(const char* msg, va_list ap); + +/// Log an informational message. +void Info(const char* msg, ...); +void Info(const char* msg, va_list ap); /// Canonicalize a path like "foo/../bar.h" into just "bar.h". /// |slash_bits| has bits set starting from lowest for a backslash that was /// normalized to a forward slash. (only used on Windows) -bool CanonicalizePath(string* path, uint64_t* slash_bits, string* err); -bool CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits, - string* err); +void CanonicalizePath(std::string* path, uint64_t* slash_bits); +void CanonicalizePath(char* path, size_t* len, uint64_t* slash_bits); /// Appends |input| to |*result|, escaping according to the whims of either /// Bash, or Win32's CommandLineToArgvW(). /// Appends the string directly to |result| without modification if we can /// determine that it contains no problematic characters. -void GetShellEscapedString(const string& input, string* result); -void GetWin32EscapedString(const string& input, string* result); +void GetShellEscapedString(const std::string& input, std::string* result); +void GetWin32EscapedString(const std::string& input, std::string* result); /// Read a file to a string (in text mode: with CRLF conversion /// on Windows). /// Returns -errno and fills in \a err on error. -int ReadFile(const string& path, string* contents, string* err); +int ReadFile(const std::string& path, std::string* contents, std::string* err); /// Mark a file descriptor to not be inherited on exec()s. void SetCloseOnExec(int fd); /// Given a misspelled string and a list of correct spellings, returns /// the closest match or NULL if there is no close enough match. -const char* SpellcheckStringV(const string& text, - const vector& words); +const char* SpellcheckStringV(const std::string& text, + const std::vector& words); /// Like SpellcheckStringV, but takes a NULL-terminated list. const char* SpellcheckString(const char* text, ...); @@ -87,7 +92,7 @@ const char* SpellcheckString(const char* text, ...); bool islatinalpha(int c); /// Removes all Ansi escape codes (http://www.termsys.demon.co.uk/vtansi.htm). -string StripAnsiEscapeCodes(const string& in); +std::string StripAnsiEscapeCodes(const std::string& in); /// @return the number of processors on the machine. Useful for an initial /// guess for how many jobs to run in parallel. @return 0 on error. @@ -97,17 +102,15 @@ int GetProcessorCount(); /// on error. double GetLoadAverage(); -/// Elide the given string @a str with '...' in the middle if the length -/// exceeds @a width. -string ElideMiddle(const string& str, size_t width); +/// a wrapper for getcwd() +std::string GetWorkingDirectory(); /// Truncates a file to the given size. -bool Truncate(const string& path, size_t size, string* err); +bool Truncate(const std::string& path, size_t size, std::string* err); #ifdef _MSC_VER #define snprintf _snprintf #define fileno _fileno -#define unlink _unlink #define chdir _chdir #define strtoull _strtoui64 #define getcwd _getcwd @@ -116,10 +119,22 @@ bool Truncate(const string& path, size_t size, string* err); #ifdef _WIN32 /// Convert the value returned by GetLastError() into a string. -string GetLastErrorString(); +std::string GetLastErrorString(); /// Calls Fatal() with a function name and GetLastErrorString. NORETURN void Win32Fatal(const char* function, const char* hint = NULL); + +/// Naive implementation of C++ 20 std::bit_cast(), used to fix Clang and GCC +/// [-Wcast-function-type] warning on casting result of GetProcAddress(). +template +inline To FunctionCast(From from) { + static_assert(sizeof(To) == sizeof(From), ""); + To result; + memcpy(&result, &from, sizeof(To)); + return result; +} #endif +int platformAwareUnlink(const char* filename); + #endif // NINJA_UTIL_H_ diff --git a/src/util_test.cc b/src/util_test.cc index d97b48ccc2..38e65e9338 100644 --- a/src/util_test.cc +++ b/src/util_test.cc @@ -16,246 +16,293 @@ #include "test.h" +using namespace std; + namespace { -bool CanonicalizePath(string* path, string* err) { +void CanonicalizePath(string* path) { uint64_t unused; - return ::CanonicalizePath(path, &unused, err); + ::CanonicalizePath(path, &unused); } } // namespace TEST(CanonicalizePath, PathSamples) { string path; - string err; - EXPECT_FALSE(CanonicalizePath(&path, &err)); - EXPECT_EQ("empty path", err); + CanonicalizePath(&path); + EXPECT_EQ("", path); - path = "foo.h"; err = ""; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + path = "foo.h"; + CanonicalizePath(&path); EXPECT_EQ("foo.h", path); path = "./foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo.h", path); path = "./foo/./bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo/bar.h", path); path = "./x/foo/../bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("x/bar.h", path); path = "./x/foo/../../bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("bar.h", path); path = "foo//bar"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo/bar", path); path = "foo//.//..///bar"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("bar", path); path = "./x/../foo/../../bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("../bar.h", path); path = "foo/./."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo", path); path = "foo/bar/.."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo", path); path = "foo/.hidden_bar"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo/.hidden_bar", path); path = "/foo"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("/foo", path); path = "//foo"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); #ifdef _WIN32 EXPECT_EQ("//foo", path); #else EXPECT_EQ("/foo", path); #endif + path = ".."; + CanonicalizePath(&path); + EXPECT_EQ("..", path); + + path = "../"; + CanonicalizePath(&path); + EXPECT_EQ("..", path); + + path = "../foo"; + CanonicalizePath(&path); + EXPECT_EQ("../foo", path); + + path = "../foo/"; + CanonicalizePath(&path); + EXPECT_EQ("../foo", path); + + path = "../.."; + CanonicalizePath(&path); + EXPECT_EQ("../..", path); + + path = "../../"; + CanonicalizePath(&path); + EXPECT_EQ("../..", path); + + path = "./../"; + CanonicalizePath(&path); + EXPECT_EQ("..", path); + + path = "/.."; + CanonicalizePath(&path); + EXPECT_EQ("/..", path); + + path = "/../"; + CanonicalizePath(&path); + EXPECT_EQ("/..", path); + + path = "/../.."; + CanonicalizePath(&path); + EXPECT_EQ("/../..", path); + + path = "/../../"; + CanonicalizePath(&path); + EXPECT_EQ("/../..", path); + path = "/"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); - EXPECT_EQ("", path); + CanonicalizePath(&path); + EXPECT_EQ("/", path); path = "/foo/.."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); - EXPECT_EQ("", path); + CanonicalizePath(&path); + EXPECT_EQ("/", path); path = "."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ(".", path); path = "./."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ(".", path); path = "foo/.."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ(".", path); + + path = "foo/.._bar"; + CanonicalizePath(&path); + EXPECT_EQ("foo/.._bar", path); } #ifdef _WIN32 TEST(CanonicalizePath, PathSamplesWindows) { string path; - string err; - EXPECT_FALSE(CanonicalizePath(&path, &err)); - EXPECT_EQ("empty path", err); + CanonicalizePath(&path); + EXPECT_EQ("", path); - path = "foo.h"; err = ""; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + path = "foo.h"; + CanonicalizePath(&path); EXPECT_EQ("foo.h", path); path = ".\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo.h", path); path = ".\\foo\\.\\bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo/bar.h", path); path = ".\\x\\foo\\..\\bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("x/bar.h", path); path = ".\\x\\foo\\..\\..\\bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("bar.h", path); path = "foo\\\\bar"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo/bar", path); path = "foo\\\\.\\\\..\\\\\\bar"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("bar", path); path = ".\\x\\..\\foo\\..\\..\\bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("../bar.h", path); path = "foo\\.\\."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo", path); path = "foo\\bar\\.."; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo", path); path = "foo\\.hidden_bar"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("foo/.hidden_bar", path); path = "\\foo"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("/foo", path); path = "\\\\foo"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("//foo", path); path = "\\"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); - EXPECT_EQ("", path); + CanonicalizePath(&path); + EXPECT_EQ("/", path); } TEST(CanonicalizePath, SlashTracking) { string path; - string err; uint64_t slash_bits; - path = "foo.h"; err = ""; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + path = "foo.h"; + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("foo.h", path); EXPECT_EQ(0, slash_bits); path = "a\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/foo.h", path); EXPECT_EQ(1, slash_bits); path = "a/bcd/efh\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/bcd/efh/foo.h", path); EXPECT_EQ(4, slash_bits); path = "a\\bcd/efh\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/bcd/efh/foo.h", path); EXPECT_EQ(5, slash_bits); path = "a\\bcd\\efh\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/bcd/efh/foo.h", path); EXPECT_EQ(7, slash_bits); path = "a/bcd/efh/foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/bcd/efh/foo.h", path); EXPECT_EQ(0, slash_bits); path = "a\\./efh\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/efh/foo.h", path); EXPECT_EQ(3, slash_bits); path = "a\\../efh\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("efh/foo.h", path); EXPECT_EQ(1, slash_bits); path = "a\\b\\c\\d\\e\\f\\g\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/b/c/d/e/f/g/foo.h", path); EXPECT_EQ(127, slash_bits); path = "a\\b\\c\\..\\..\\..\\g\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("g/foo.h", path); EXPECT_EQ(1, slash_bits); path = "a\\b/c\\../../..\\g\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("g/foo.h", path); EXPECT_EQ(1, slash_bits); path = "a\\b/c\\./../..\\g\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/g/foo.h", path); EXPECT_EQ(3, slash_bits); path = "a\\b/c\\./../..\\g/foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/g/foo.h", path); EXPECT_EQ(1, slash_bits); path = "a\\\\\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/foo.h", path); EXPECT_EQ(1, slash_bits); path = "a/\\\\foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/foo.h", path); EXPECT_EQ(0, slash_bits); path = "a\\//foo.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ("a/foo.h", path); EXPECT_EQ(1, slash_bits); } @@ -264,22 +311,20 @@ TEST(CanonicalizePath, CanonicalizeNotExceedingLen) { // Make sure searching \/ doesn't go past supplied len. char buf[] = "foo/bar\\baz.h\\"; // Last \ past end. uint64_t slash_bits; - string err; size_t size = 13; - EXPECT_TRUE(::CanonicalizePath(buf, &size, &slash_bits, &err)); + ::CanonicalizePath(buf, &size, &slash_bits); EXPECT_EQ(0, strncmp("foo/bar/baz.h", buf, size)); EXPECT_EQ(2, slash_bits); // Not including the trailing one. } TEST(CanonicalizePath, TooManyComponents) { string path; - string err; uint64_t slash_bits; // 64 is OK. path = "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./" "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./x.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ(slash_bits, 0x0); // Backslashes version. @@ -289,83 +334,125 @@ TEST(CanonicalizePath, TooManyComponents) { "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\" "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\x.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ(slash_bits, 0xffffffff); // 65 is OK if #component is less than 60 after path canonicalization. - err = ""; path = "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./" "a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./a/./x/y.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ(slash_bits, 0x0); // Backslashes version. - err = ""; path = "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\" "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\" "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\" "a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\a\\.\\x\\y.h"; - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); - EXPECT_EQ(slash_bits, 0x1ffffffff); + CanonicalizePath(&path, &slash_bits); + EXPECT_EQ(slash_bits, uint64_t(0x1ffffffff)); // 59 after canonicalization is OK. - err = ""; path = "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/x/y.h"; EXPECT_EQ(58, std::count(path.begin(), path.end(), '/')); - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); + CanonicalizePath(&path, &slash_bits); EXPECT_EQ(slash_bits, 0x0); // Backslashes version. - err = ""; path = "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" "a\\a\\a\\a\\a\\a\\a\\a\\a\\x\\y.h"; EXPECT_EQ(58, std::count(path.begin(), path.end(), '\\')); - EXPECT_TRUE(CanonicalizePath(&path, &slash_bits, &err)); - EXPECT_EQ(slash_bits, 0x3ffffffffffffff); + CanonicalizePath(&path, &slash_bits); + EXPECT_EQ(slash_bits, uint64_t(0x3ffffffffffffff)); + + // More than 60 components is now completely ok too. + path = + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\a\\" + "a\\a\\a\\a\\a\\a\\a\\a\\a\\x\\y.h"; + EXPECT_EQ(218, std::count(path.begin(), path.end(), '\\')); + CanonicalizePath(&path, &slash_bits); + EXPECT_EQ(slash_bits, 0xffffffffffffffff); } -#endif +#else // !_WIN32 +TEST(CanonicalizePath, TooManyComponents) { + string path; + uint64_t slash_bits; + + // More than 60 components is now completely ok. + path = + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/a/" + "a/a/a/a/a/a/a/a/a/x/y.h"; + EXPECT_EQ(218, std::count(path.begin(), path.end(), '/')); + CanonicalizePath(&path, &slash_bits); + EXPECT_EQ(slash_bits, 0x0); +} +#endif // !_WIN32 TEST(CanonicalizePath, UpDir) { string path, err; path = "../../foo/bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("../../foo/bar.h", path); path = "test/../../foo/bar.h"; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("../foo/bar.h", path); } TEST(CanonicalizePath, AbsolutePath) { string path = "/usr/include/stdio.h"; string err; - EXPECT_TRUE(CanonicalizePath(&path, &err)); + CanonicalizePath(&path); EXPECT_EQ("/usr/include/stdio.h", path); } TEST(CanonicalizePath, NotNullTerminated) { string path; - string err; size_t len; uint64_t unused; path = "foo/. bar/."; len = strlen("foo/."); // Canonicalize only the part before the space. - EXPECT_TRUE(CanonicalizePath(&path[0], &len, &unused, &err)); + CanonicalizePath(&path[0], &len, &unused); EXPECT_EQ(strlen("foo"), len); EXPECT_EQ("foo/. bar/.", string(path)); + // Verify that foo/..file gets canonicalized to 'file' without + // touching the rest of the string. path = "foo/../file bar/."; len = strlen("foo/../file"); - EXPECT_TRUE(CanonicalizePath(&path[0], &len, &unused, &err)); + CanonicalizePath(&path[0], &len, &unused); EXPECT_EQ(strlen("file"), len); - EXPECT_EQ("file ./file bar/.", string(path)); + EXPECT_EQ("file../file bar/.", string(path)); } TEST(PathEscaping, TortureTest) { @@ -415,16 +502,3 @@ TEST(StripAnsiEscapeCodes, StripColors) { EXPECT_EQ("affixmgr.cxx:286:15: warning: using the result... [-Wparentheses]", stripped); } - -TEST(ElideMiddle, NothingToElide) { - string input = "Nothing to elide in this short string."; - EXPECT_EQ(input, ElideMiddle(input, 80)); - EXPECT_EQ(input, ElideMiddle(input, 38)); -} - -TEST(ElideMiddle, ElideInTheMiddle) { - string input = "01234567890123456789"; - string elided = ElideMiddle(input, 10); - EXPECT_EQ("012...789", elided); - EXPECT_EQ("01234567...23456789", ElideMiddle(input, 19)); -} diff --git a/src/version.cc b/src/version.cc index 1c906aee06..f97b77d568 100644 --- a/src/version.cc +++ b/src/version.cc @@ -18,7 +18,9 @@ #include "util.h" -const char* kNinjaVersion = "1.9.0.git"; +using namespace std; + +const char* kNinjaVersion = "1.13.0.git"; void ParseVersion(const string& version, int* major, int* minor) { size_t end = version.find('.'); diff --git a/src/version.h b/src/version.h index bd6b9ffe21..9d84ecb1e9 100644 --- a/src/version.h +++ b/src/version.h @@ -16,17 +16,16 @@ #define NINJA_VERSION_H_ #include -using namespace std; /// The version number of the current Ninja release. This will always /// be "git" on trunk. extern const char* kNinjaVersion; /// Parse the major/minor components of a version string. -void ParseVersion(const string& version, int* major, int* minor); +void ParseVersion(const std::string& version, int* major, int* minor); /// Check whether \a version is compatible with the current Ninja version, /// aborting if not. -void CheckNinjaVersion(const string& required_version); +void CheckNinjaVersion(const std::string& required_version); #endif // NINJA_VERSION_H_ diff --git a/windows/ninja.manifest b/windows/ninja.manifest new file mode 100644 index 0000000000..47949dd7ce --- /dev/null +++ b/windows/ninja.manifest @@ -0,0 +1,9 @@ + + + + + UTF-8 + true + + +