Remove unneeded include, improve Memento code, add Jump CH

Author: Amos Brocco

CMakeLists.txt

@@ -32,6 +32,7 @@ add_executable(speed_test speed_test.cpp
     anchor/AnchorHashQre.cpp anchor/AnchorHashQre.hpp  anchor/misc/crc32c_sse42_u64.h
     anchor/anchorengine.h
     memento/mashtable.h
+    jump/jumpengine.h
     )
 
 add_executable(balance balance.cpp
@@ -41,12 +42,15 @@ add_executable(balance balance.cpp
     anchor/AnchorHashQre.cpp anchor/AnchorHashQre.hpp  anchor/misc/crc32c_sse42_u64.h
     anchor/anchorengine.h
     memento/mashtable.h
+    jump/jumpengine.h
     )
 
 add_executable(mashtable_test mashtable_test.cpp memento/mashtable.h)
 
-target_include_directories(speed_test PRIVATE ${PCG_INCLUDE_DIRS})
-target_include_directories(balance PRIVATE ${PCG_INCLUDE_DIRS})
+if(WITH_PCG32)
+    target_include_directories(speed_test PRIVATE ${PCG_INCLUDE_DIRS})
+    target_include_directories(balance PRIVATE ${PCG_INCLUDE_DIRS})
+endif()
 target_link_libraries(speed_test PRIVATE xxHash::xxhash fmt::fmt cxxopts::cxxopts)
 target_link_libraries(balance PRIVATE xxHash::xxhash fmt::fmt cxxopts::cxxopts)
 

balance.cpp

@@ -24,6 +24,7 @@
 #include "anchor/anchorengine.h"
 #include "memento/mashtable.h"
 #include "memento/mementoengine.h"
+#include "jump/jumpengine.h"
 #include <fmt/core.h>
 #include <fstream>
 #include <unordered_map>
@@ -124,7 +125,7 @@ int main(int argc, char *argv[]) {
   cxxopts::Options options("speed_test", "MementoHash vs AnchorHash benchmark");
   options.add_options()(
       "Algorithm",
-      "Algorithm (null|baseline|anchor|memento|mementoboost|mementomash)",
+      "Algorithm (null|baseline|anchor|memento|mementoboost|mementomash|jump)",
       cxxopts::value<std::string>())(
       "AnchorSet", "Size of the AnchorSet (ignored by Memento)",
       cxxopts::value<int>())("WorkingSet", "Size of the WorkingSet",
@@ -195,6 +196,10 @@ int main(int argc, char *argv[]) {
     return bench<MementoEngine<MashTable>>("Memento<MashTable>", filename,
                                            anchor_set, working_set,
                                            num_removals, num_keys);
+  } else if (algorithm == "jump") {
+      return bench<JumpEngine>("JumpEngine", filename,
+                               anchor_set, working_set,
+                               num_removals, num_keys);
   } else {
     fmt::println("Unknown algorithm {}", algorithm);
     return 2;

jump/jumpengine.h

@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2023 Amos Brocco.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, version 3.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef JUMPENGINE_H
+#define JUMPENGINE_H
+#include <cstdint>
+
+class JumpEngine final {
+public:
+    JumpEngine(uint32_t, uint32_t working_set)
+        : m_num_buckets{working_set}
+    {}
+
+    // From AnchorHash
+    static uint32_t crc32c_sse42_u64(uint64_t key, uint64_t seed) {
+        __asm__ volatile("crc32q %[key], %[seed];"
+                         : [seed] "+r"(seed)
+                         : [key] "rm"(key));
+        return seed;
+    }
+
+    /**
+   * Returns the bucket where the given key should be mapped.
+   * This implementations is the same as provided by Jump authors
+   *
+   * @param key the key to map
+   * @param key the initial seed for CRC32c
+   * @return the related bucket
+   */
+    uint32_t getBucketCRC32c(uint64_t key, uint64_t seed) noexcept
+    {
+        uint64_t hash = crc32c_sse42_u64(key, seed);
+        int64_t b = 1, j = 0;
+        while (j < m_num_buckets) {
+            b = j;
+            hash = hash * 2862933555777941757ULL + 1;
+            j = (b + 1) * (double(1LL << 31) / double((hash >> 33) + 1));
+        }
+        return b;
+    }
+
+    /**
+   * Adds a new bucket to the engine.
+   *
+   * @return the added bucket
+   */
+    uint32_t addBucket() noexcept { return m_num_buckets++; }
+
+    /**
+   * Removes the given bucket from the engine.
+   * Since Jump does not support random removals, it will always remove the
+   * last bucket.
+   *
+   * @return the removed bucket
+   */
+    uint32_t removeBucket(uint32_t) noexcept
+    {
+        return --m_num_buckets;
+    }
+
+private:
+    uint32_t m_num_buckets;
+};
+
+#endif // JUMPENGINE_H

memento/memento.h

@@ -16,8 +16,6 @@
  * You should have received a copy of the GNU General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include <optional>
-#include <boost/unordered/unordered_flat_map.hpp>
 
 template<template <typename...> class MementoMap, typename... Args>
 class Memento final
@@ -36,23 +34,24 @@ class Memento final
     };
 
     MementoMap<uint32_t, Entry> m_table;
+    uint32_t m_table_size;
 
 public:
-    Memento();
+    Memento() {}
 
     /**
      * Returns the size of the replacement set.
      *
      * @return the size of the replacement set
      */
-    int32_t size() const noexcept;
+    int32_t size() const noexcept { return m_table_size; }
 
     /**
      * Returns {@code true} if the replacement set is empty.
      *
      * @return {@code true} if empty, {@code false} otherwise
      */
-    bool isEmpty() const noexcept;
+    bool isEmpty() const noexcept { return m_table_size == 0; }
 
     /**
      * Remembers that the given bucket has been removed
@@ -67,7 +66,11 @@ class Memento final
      * @param prevRemoved the previous removed bucket
      * @return the value of the new last removed bucket
      */
-    int32_t remember(uint32_t bucket, uint32_t replacer, uint32_t prevRemoved ) noexcept;
+    int32_t remember(uint32_t bucket, uint32_t replacer, uint32_t prevRemoved ) noexcept {
+        m_table.emplace(bucket, Entry{replacer, prevRemoved});
+        ++m_table_size;
+        return bucket;
+    }
 
     /**
      * Restores the given bucket by removing it
@@ -79,7 +82,15 @@ class Memento final
      * @param bucket the bucket to restore
      * @return the new last removed bucket
      */
-    int32_t restore(uint32_t bucket) noexcept;
+    int32_t restore(uint32_t bucket) noexcept {
+        if (m_table_size == 0) {
+            return bucket + 1;
+        }
+        auto e = m_table.find(bucket);
+        m_table.erase(e);
+        --m_table_size;
+        return e->second.prevRemoved;
+    }
 
     /**
      * Returns the replacer of the bucket if it
@@ -93,87 +104,16 @@ class Memento final
      * @param bucket the bucket to search for
      * @return the replacing bucket if any, {@code std::nullopt} otherwise
      */
-    std::optional<int32_t> replacer(int32_t bucket ) const noexcept;
+    int32_t replacer(int32_t bucket ) const noexcept {
+        if (m_table_size == 0) {
+            return -1;
+        }
+        auto e = m_table.find(bucket);
+        if (e != m_table.end()) {
+            return e->second.replacer;
+        } else {
+            return -1;
+        }
+    }
 };
-
-template<template <typename...> class MementoMap, typename... Args>
-    Memento<MementoMap,Args...>::Memento() {}
-
-/**
- * Returns the size of the replacement set.
- *
- * @return the size of the replacement set
- */
-template<template <typename...> class MementoMap, typename... Args>
-inline int32_t Memento<MementoMap,Args...>::size() const noexcept { return m_table.size(); }
-
-/**
- * Returns {@code true} if the replacement set is empty.
- *
- * @return {@code true} if empty, {@code false} otherwise
- */
-template<template <typename...> class MementoMap, typename... Args>
-inline bool Memento<MementoMap,Args...>::isEmpty() const noexcept { return m_table.empty(); }
-
-/**
- * Remembers that the given bucket has been removed
- * and that was replaced by the given replacer.
- * <p>
- * This method also stores the last removed bucket
- * (before the current one) to create the sequence
- * of removals.
- *
- * @param bucket      the removed bucket
- * @param replacer    the replacing bucket
- * @param prevRemoved the previous removed bucket
- * @return the value of the new last removed bucket
- */
-template<template <typename...> class MementoMap, typename... Args>
-inline int32_t Memento<MementoMap,Args...>::remember(uint32_t bucket, uint32_t replacer,
-                          uint32_t prevRemoved) noexcept {
-  m_table.emplace(bucket, Entry{replacer, prevRemoved});
-  return bucket;
-}
-
-/**
- * Restores the given bucket by removing it
- * from the memory.
- * <p>
- * If the memory is empty the last removed bucket
- * becomes the given bucket + 1.
- *
- * @param bucket the bucket to restore
- * @return the new last removed bucket
- */
-template<template <typename...> class MementoMap, typename... Args>
-inline int32_t Memento<MementoMap,Args...>::restore(uint32_t bucket) noexcept {
-  if (m_table.empty()) {
-    return bucket + 1;
-  }
-  auto e = m_table.find(bucket);
-  m_table.erase(e);
-  return e->second.prevRemoved;
-}
-
-/**
- * Returns the replacer of the bucket if it
- * was removed, otherwise returns {@code -1}.
- * <p>
- * The value returned by this method represents
- * both the bucket that replaced the given one
- * and the size of the working set after removing
- * the given bucket.
- *
- * @param bucket the bucket to search for
- * @return the replacing bucket if any, {@code std::nullopt} otherwise
- */
-template<template <typename...> class MementoMap, typename... Args>
-inline std::optional<int32_t> Memento<MementoMap,Args...>::replacer(int32_t bucket) const noexcept {
-  auto e = m_table.find(bucket);
-  if (e != m_table.end()) {
-    return e->second.replacer;
-  } else {
-    return std::nullopt;
-  }
-}
 #endif // MEMENTO_H

memento/mementoengine.h

@@ -16,7 +16,6 @@
 #ifndef MEMENTOENGINE_H
 #define MEMENTOENGINE_H
 #include "memento.h"
-#include <cassert>
 #include <string_view>
 #include <xxhash.h>
 
@@ -116,20 +115,19 @@ MementoEngine<MementoMap, Args...>::MementoEngine(uint32_t, uint32_t size)
 template <template <typename...> class MementoMap, typename... Args>
 uint32_t MementoEngine<MementoMap, Args...>::getBucket(
     std::string_view key) const noexcept {
-  assert(m_bArraySize > 0);
   /*
    * We invoke JumpHash to get a bucket
    * in the range [0,bArraySize-1].
    */
-  auto hash{XXH64(key.data(), key.size(), 0)};
+  const auto hash{XXH64(key.data(), key.size(), 0)};
   auto b = JumpConsistentHash(hash, m_bArraySize);
 
   /*
    * We check if the bucket was removed, if not we are done.
    * If the bucket was removed the replacing bucket is >= 0,
    * otherwise it is -1.
    */
-  auto replacer = m_memento.replacer(b).value_or(-1);
+  auto replacer = m_memento.replacer(b);
   while (replacer >= 0) {
 
     /*
@@ -139,18 +137,18 @@ uint32_t MementoEngine<MementoMap, Args...>::getBucket(
      * represents the size of the working set when the bucket
      * was removed and get a new bucket in [0,replacer-1].
      */
-    auto h = XXH64(key.data(), key.size(), b);
+    const auto h = XXH64(key.data(), key.size(), b);
     b = h % replacer;
 
     /*
      * If we hit a removed bucket we follow the replacements
      * until we get a working bucket or a bucket in the range
      * [0,replacer-1]
      */
-    auto r = m_memento.replacer(b).value_or(-1);
+    auto r = m_memento.replacer(b);
     while (r >= replacer) {
       b = r;
-      r = m_memento.replacer(b).value_or(-1);
+      r = m_memento.replacer(b);
     }
 
     /* Finally we update the entry of the external loop. */
@@ -163,9 +161,7 @@ uint32_t MementoEngine<MementoMap, Args...>::getBucket(
 template <template <typename...> class MementoMap, typename... Args>
 uint32_t MementoEngine<MementoMap, Args...>::getBucketCRC32c(
     uint64_t key, uint64_t seed) const noexcept {
-  assert(m_bArraySize > 0);
-
-  uint32_t hash = crc32c_sse42_u64(key, seed);
+  const auto hash = crc32c_sse42_u64(key, seed);
   /*
    * We invoke JumpHash to get a bucket
    * in the range [0,bArraySize-1].
@@ -177,7 +173,7 @@ uint32_t MementoEngine<MementoMap, Args...>::getBucketCRC32c(
    * If the bucket was removed the replacing bucket is >= 0,
    * otherwise it is -1.
    */
-  auto replacer = m_memento.replacer(b).value_or(-1);
+  auto replacer = m_memento.replacer(b);
   while (replacer >= 0) {
 
     /*
@@ -187,18 +183,18 @@ uint32_t MementoEngine<MementoMap, Args...>::getBucketCRC32c(
      * represents the size of the working set when the bucket
      * was removed and get a new bucket in [0,replacer-1].
      */
-    auto h = crc32c_sse42_u64(key, b);
+    const auto h = crc32c_sse42_u64(key, b);
     b = h % replacer;
 
     /*
      * If we hit a removed bucket we follow the replacements
      * until we get a working bucket or a bucket in the range
      * [0,replacer-1]
      */
-    auto r = m_memento.replacer(b).value_or(-1);
+    auto r = m_memento.replacer(b);
     while (r >= replacer) {
       b = r;
-      r = m_memento.replacer(b).value_or(-1);
+        r = m_memento.replacer(b);
     }
 
     /* Finally we update the entry of the external loop. */