Introduce config.h to control feature selection, and ENABLE_HMEM option

This is Weijia's idea, but I'm copying it into a new commit so it can be
in a separate pull request from the Persistent API changes. The
ENABLE_HMEM option is critical to allow Derecho to run on libfabrics'
tcp provider. The config.h file puts these compile-time feature
selection macros into a header file instead of expecting them to be
passed as -D options in the compiler flags.

CMakeLists.txt

@@ -8,9 +8,6 @@ set(derecho_VERSION 2.4)
 set(derecho_build_VERSION 2.4.0)
 set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-if (${USE_VERBS_API})
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_VERBS_API")
-endif()
 set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -DDERECHO_DEBUG -O0 -Wall -ggdb -gdwarf-3")
 set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -Wall")
 set(CMAKE_CXX_FLAGS_BENCHMARK "${CMAKE_CXX_FLAGS_RELEASE} -Wall -DNOLOG -Ofast -march=native")
@@ -62,6 +59,25 @@ find_package(nlohmann_json 3.9.0 REQUIRED)
 # Target: Doxygen
 find_package(Doxygen)
 
+
+# Source code configurations go to config.h
+# Shift to verbs API. We use libfabric by default.
+if (${USE_VERBS_API})
+set(USE_VERBS_API 1)
+else()
+set(USE_VERBS_API 0)
+endif()
+# Enable High memory registration (for device memory like GPU memory.)
+# Important: this does not work with libfabric provider like tcp and socket because they cannot
+# RDMA to device memory.
+if (${ENABLE_HMEM})
+set(ENABLE_HMEM 1)
+else()
+set(ENABLE_HMEM 0)
+endif()
+CONFIGURE_FILE(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/derecho/config.h)
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/include)
+
 add_subdirectory(src/mutils-serialization)
 add_subdirectory(src/conf)
 add_subdirectory(src/utils)
@@ -118,8 +134,9 @@ install(TARGETS derecho
         INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 # Declare that we will install the "include/" directory as a standard include directory
 install(DIRECTORY
-        include/
-        TYPE INCLUDE)
+        ${CMAKE_CURRENT_SOURCE_DIR}/include/derecho
+        ${CMAKE_CURRENT_BINARY_DIR}/include/derecho
+        DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 
 # Use CMakePackageConfigHelpers to create a package version file and config file
 include(CMakePackageConfigHelpers)

config.h.in

@@ -0,0 +1,3 @@
+#pragma once
+#cmakedefine    USE_VERBS_API
+#cmakedefine    ENABLE_HMEM

include/derecho/conf/conf.hpp

@@ -1,7 +1,10 @@
+#pragma once
 #ifndef CONF_HPP
 #define CONF_HPP
 
+#include "derecho/config.h"
 #include "getpot/GetPot"
+
 #include <atomic>
 #include <getopt.h>
 #include <inttypes.h>

include/derecho/core/bytes_object.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "derecho/config.h"
 #include "derecho/mutils-serialization/SerializationSupport.hpp"
 
 #include <cstring>

include/derecho/core/derecho.hpp

@@ -5,6 +5,7 @@
  * @brief   This file includes the essential core headers.
  */
 
+#include "derecho/config.h"
 #include "derecho_exception.hpp"
 #include "detail/derecho_internal.hpp"
 #include "derecho_type_definitions.hpp"

include/derecho/core/derecho_exception.hpp

@@ -5,6 +5,7 @@
 
 #pragma once
 
+#include "derecho/config.h"
 #include "derecho/core/git_version.hpp"
 
 #include <exception>

include/derecho/core/derecho_type_definitions.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "derecho/config.h"
+
 #include <string>
 #include <cstdint>
 

include/derecho/core/detail/connection_manager.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "../derecho_type_definitions.hpp"
+#include "derecho/config.h"
 #include "derecho/tcp/tcp.hpp"
 #include "locked_reference.hpp"
 

include/derecho/core/detail/derecho_sst.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "../derecho_type_definitions.hpp"
+#include "derecho/config.h"
 #include "derecho/sst/sst.hpp"
 #include "derecho_internal.hpp"
 

include/derecho/core/detail/external_group_impl.hpp

@@ -1,4 +1,5 @@
 #include "../external_group.hpp"
+#include "derecho/config.h"
 #include "version_code.hpp"
 
 namespace derecho {

include/derecho/core/detail/group_impl.hpp

@@ -4,6 +4,7 @@
  */
 
 #include "../group.hpp"
+#include "derecho/config.h"
 #include "derecho/mutils-serialization/SerializationSupport.hpp"
 #include "derecho/utils/container_template_functions.hpp"
 #include "derecho/utils/logger.hpp"
@@ -28,7 +29,7 @@ template <typename SubgroupType>
 uint32_t _Group::get_subgroup_max_payload_size(uint32_t subgroup_num) {
     if (auto gptr = dynamic_cast<GroupProjection<SubgroupType>*>(this)) {
         return gptr->get_subgroup_max_payload_size(subgroup_num);
-    } else 
+    } else
         throw derecho_exception("Error: this top-level group contains no subgroups for the selected type.");
 }
 

include/derecho/core/detail/make_kind_map.hpp

@@ -8,6 +8,7 @@
 
 #pragma once
 
+#include "derecho/config.h"
 #include "derecho_internal.hpp"
 #include <mutils-containers/KindMap.hpp>
 

include/derecho/core/detail/multicast_group.hpp

@@ -4,6 +4,7 @@
 #include "../subgroup_info.hpp"
 #include "connection_manager.hpp"
 #include "derecho/conf/conf.hpp"
+#include "derecho/config.h"
 #include "derecho/mutils-serialization/SerializationMacros.hpp"
 #include "derecho/mutils-serialization/SerializationSupport.hpp"
 #include "derecho/rdmc/rdmc.hpp"

include/derecho/core/detail/p2p_connection.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "derecho/config.h"
 #ifdef USE_VERBS_API
 #include "derecho/sst/detail/verbs.hpp"
 #else

include/derecho/core/detail/p2p_connection_manager.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "derecho/config.h"
 #include "p2p_connection.hpp"
 #ifdef USE_VERBS_API
 #include "derecho/sst/detail/verbs.hpp"

include/derecho/core/detail/persistence_manager.hpp

@@ -5,6 +5,7 @@
  */
 #pragma once
 
+#include "derecho/config.h"
 #include "derecho/openssl/signature.hpp"
 #include "derecho/persistent/PersistentInterface.hpp"
 #include "derecho/utils/logger.hpp"

include/derecho/core/detail/public_key_store.hpp

@@ -4,6 +4,7 @@
 #pragma once
 
 #include "../derecho_exception.hpp"
+#include "derecho/config.h"
 #include "derecho/openssl/signature.hpp"
 #include "derecho_internal.hpp"
 

include/derecho/core/detail/remote_invocable.hpp

@@ -7,6 +7,7 @@
 #pragma once
 
 #include "derecho/conf/conf.hpp"
+#include "derecho/config.h"
 #include "derecho/mutils-serialization/SerializationSupport.hpp"
 #include "derecho/utils/logger.hpp"
 #include "rpc_utils.hpp"

include/derecho/core/detail/replicated_impl.hpp

@@ -1,6 +1,7 @@
 #pragma once
 
 #include "../replicated.hpp"
+#include "derecho/config.h"
 #include "view_manager.hpp"
 
 #include "derecho/mutils-serialization/SerializationSupport.hpp"

include/derecho/core/detail/replicated_interface.hpp

@@ -1,5 +1,6 @@
 #pragma once
 
+#include "derecho/config.h"
 #include "derecho/openssl/signature.hpp"
 #include "derecho/tcp/tcp.hpp"
 #include "derecho_internal.hpp"

include/derecho/core/detail/restart_state.hpp

@@ -2,8 +2,9 @@
 
 #include "../subgroup_info.hpp"
 #include "../view.hpp"
-#include "derecho_internal.hpp"
+#include "derecho/config.h"
 #include "derecho/mutils-serialization/SerializationSupport.hpp"
+#include "derecho_internal.hpp"
 
 #include <spdlog/spdlog.h>
 

include/derecho/core/detail/rpc_manager.hpp

@@ -8,6 +8,7 @@
 
 #include "../derecho_type_definitions.hpp"
 #include "../view.hpp"
+#include "derecho/config.h"
 #include "derecho/mutils-serialization/SerializationSupport.hpp"
 #include "derecho/persistent/Persistent.hpp"
 #include "derecho/utils/logger.hpp"
@@ -281,11 +282,11 @@ class RPCManager {
      * are set up in the new-view callback, but external clients can join at any time.
      */
     void add_external_connection(node_id_t node_id);
-    
+
     /**
-     * Remove a P2P connection to an external client. The connection is cleaned up in 
-     * case a failure is detected (i.e. the external client stopped sending heartbeats. 
-     * However, we avoid the failure procedure in case the external client performs a 
+     * Remove a P2P connection to an external client. The connection is cleaned up in
+     * case a failure is detected (i.e. the external client stopped sending heartbeats.
+     * However, we avoid the failure procedure in case the external client performs a
      * graceful exit, informing that it is exiting.
      */
     void remove_external_connection(node_id_t node_id);

include/derecho/core/detail/rpc_utils.hpp

@@ -8,6 +8,7 @@
 
 #include "../derecho_exception.hpp"
 #include "../derecho_type_definitions.hpp"
+#include "derecho/config.h"
 #include "derecho/mutils-serialization/SerializationSupport.hpp"
 #include "derecho/utils/logger.hpp"
 #include "derecho_internal.hpp"

include/derecho/core/detail/view_manager.hpp

@@ -8,6 +8,7 @@
 #include "../subgroup_info.hpp"
 #include "../view.hpp"
 #include "derecho/conf/conf.hpp"
+#include "derecho/config.h"
 #include "derecho/mutils-serialization/SerializationSupport.hpp"
 #include "derecho_internal.hpp"
 #include "locked_reference.hpp"

include/derecho/core/external_group.hpp

@@ -6,6 +6,7 @@
  */
 
 #include "derecho/conf/conf.hpp"
+#include "derecho/config.h"
 #include "detail/connection_manager.hpp"
 #include "detail/p2p_connection_manager.hpp"
 #include "group.hpp"

include/derecho/core/group.hpp

@@ -6,6 +6,7 @@
  */
 
 #include "derecho/conf/conf.hpp"
+#include "derecho/config.h"
 #include "derecho/tcp/tcp.hpp"
 #include "derecho_exception.hpp"
 #include "detail/derecho_internal.hpp"
@@ -546,7 +547,7 @@ class Group : public virtual _Group, public GroupProjection<ReplicatedTypes>...
      * @brief Register Out-of-band memory region with extended arguments
      * Please check fi_mr_regattr documentation for more explanation:
      * https://ofiwg.github.io/libfabric/v1.12.1/man/fi_mr.3.html
-     * 
+     *
      * @param addr      The address of the memory region
      * @param size      The size in bytes of the memory region
      * @param attr      The memory attribute

include/derecho/core/notification.hpp

@@ -5,6 +5,7 @@
  * @brief   This file include the declarations of types for the notification API.
  */
 
+#include "derecho/config.h"
 #include "derecho/mutils-serialization/SerializationMacros.hpp"
 #include "derecho/mutils-serialization/SerializationSupport.hpp"
 #include "register_rpc_functions.hpp"

include/derecho/core/register_rpc_functions.hpp

@@ -1,4 +1,6 @@
 #pragma once
+
+#include "derecho/config.h"
 #include "derecho/utils/map_macro.hpp"
 #include "detail/rpc_utils.hpp"
 

include/derecho/core/replicated.hpp

@@ -5,6 +5,7 @@
 
 #pragma once
 
+#include "derecho/config.h"
 #include "derecho/persistent/Persistent.hpp"
 #include "derecho/tcp/tcp.hpp"
 #include "derecho_exception.hpp"

include/derecho/core/subgroup_functions.hpp

@@ -4,6 +4,7 @@
 
 #pragma once
 
+#include "derecho/config.h"
 #include "derecho/utils/logger.hpp"
 #include "derecho_exception.hpp"
 #include "derecho_modes.hpp"

include/derecho/core/subgroup_info.hpp

@@ -6,6 +6,7 @@
 
 #pragma once
 
+#include "derecho/config.h"
 #include "derecho_exception.hpp"
 
 #include <cstdint>

include/derecho/core/view.hpp

@@ -5,13 +5,14 @@
 
 #pragma once
 
+#include "derecho/config.h"
+#include "derecho/mutils-serialization/SerializationMacros.hpp"
+#include "derecho/mutils-serialization/SerializationSupport.hpp"
+#include "derecho/sst/sst.hpp"
 #include "derecho_modes.hpp"
 #include "detail/derecho_internal.hpp"
 #include "detail/derecho_sst.hpp"
 #include "detail/multicast_group.hpp"
-#include "derecho/mutils-serialization/SerializationMacros.hpp"
-#include "derecho/mutils-serialization/SerializationSupport.hpp"
-#include "derecho/sst/sst.hpp"
 
 #include <cstdint>
 #include <iostream>

include/derecho/openssl/hash.hpp

@@ -5,6 +5,7 @@
  */
 #pragma once
 
+#include "derecho/config.h"
 #include "openssl_exception.hpp"
 #include "pointers.hpp"
 

include/derecho/openssl/openssl_exception.hpp

@@ -6,6 +6,8 @@
  */
 #pragma once
 
+#include "derecho/config.h"
+
 #include <exception>
 #include <iostream>
 #include <sstream>

include/derecho/openssl/pointers.hpp

@@ -6,6 +6,8 @@
 
 #pragma once
 
+#include "derecho/config.h"
+
 #include <memory>
 #include <openssl/evp.h>
 #include <openssl/bio.h>

include/derecho/openssl/signature.hpp

@@ -6,6 +6,8 @@
  */
 
 #pragma once
+
+#include "derecho/config.h"
 #include "hash.hpp"
 #include "openssl_exception.hpp"
 #include "pointers.hpp"

include/derecho/persistent/HLC.hpp

@@ -1,5 +1,7 @@
+#pragma once
 #ifndef HLC_HPP
 #define HLC_HPP
+#include "derecho/config.h"
 #include <inttypes.h>
 #include <pthread.h>
 #include <sys/types.h>

include/derecho/persistent/PersistException.hpp

@@ -1,6 +1,9 @@
+#pragma once
 #ifndef PERSISTENT_EXCEPTION_HPP
 #define PERSISTENT_EXCEPTION_HPP
 
+#include "derecho/config.h"
+
 #include <cstdint>
 #include <cstring>
 #include <stdexcept>