Running Google Test with MPI

LibTrixi.jl/src/LibTrixi.jl

@@ -1,8 +1,9 @@
 module LibTrixi
 
 using OrdinaryDiffEq: OrdinaryDiffEq, step!, check_error, DiscreteCallback
-using Trixi: Trixi, summary_callback, mesh_equations_solver_cache, nelements, nvariables,
-             nnodes, wrap_array, eachelement, cons2prim, get_node_vars, eachnode
+using Trixi: Trixi, summary_callback, mesh_equations_solver_cache, nelements,
+             nelementsglobal, nvariables, nnodes, wrap_array, eachelement, cons2prim,
+             get_node_vars, eachnode
 using MPI: MPI, run_init_hooks, set_default_error_handler_return
 using Pkg
 
@@ -27,6 +28,9 @@ export trixi_ndims,
 export trixi_nelements,
        trixi_nelements_cfptr,
        trixi_nelements_jl
+export trixi_nelements_global,
+       trixi_nelements_global_cfptr,
+       trixi_nelements_global_jl
 export trixi_nvariables,
        trixi_nvariables_cfptr,
        trixi_nvariables_jl

LibTrixi.jl/src/api_c.jl

@@ -285,7 +285,7 @@ trixi_ndims_cfptr() = @cfunction(trixi_ndims, Cint, (Cint,))
 """
     trixi_nelements(simstate_handle::Cint)::Cint
 
-Return number of elements (cells).
+Return number of local elements (cells).
 """
 function trixi_nelements end
 
@@ -297,6 +297,21 @@ end
 trixi_nelements_cfptr() = @cfunction(trixi_nelements, Cint, (Cint,))
 
 
+"""
+    trixi_nelements_global(simstate_handle::Cint)::Cint
+
+Return number of global elements (cells).
+"""
+function trixi_nelements_global end
+
+Base.@ccallable function trixi_nelements_global(simstate_handle::Cint)::Cint
+    simstate = load_simstate(simstate_handle)
+    return trixi_nelements_global_jl(simstate)
+end
+
+trixi_nelements_global_cfptr() = @cfunction(trixi_nelements_global, Cint, (Cint,))
+
+
 """
     trixi_nvariables(simstate_handle::Cint)::Cint
 

LibTrixi.jl/src/api_jl.jl

@@ -76,6 +76,12 @@ function trixi_nelements_jl(simstate)
 end
 
 
+function trixi_nelements_global_jl(simstate)
+    _, _, solver, cache = mesh_equations_solver_cache(simstate.semi)
+    return nelementsglobal(solver, cache)
+end
+
+
 function trixi_nvariables_jl(simstate)
     _, equations, _, _ = mesh_equations_solver_cache(simstate.semi)
     return nvariables(equations)

LibTrixi.jl/test/test_interface.jl

@@ -71,6 +71,10 @@ end
     nelements_jl = trixi_nelements_jl(simstate_jl)
     @test nelements_c == nelements_jl
 
+    nelements_global_c = trixi_nelements_global(handle)
+    nelements_global_jl = trixi_nelements_global_jl(simstate_jl)
+    @test nelements_global_c == nelements_global_jl
+
     # compare number of variables
     nvariables_c = trixi_nvariables(handle)
     nvariables_jl = trixi_nvariables_jl(simstate_jl)

src/api.c

@@ -13,7 +13,8 @@ enum {
     TRIXI_FTPR_STEP,
     TRIXI_FTPR_FINALIZE_SIMULATION,
     TRIXI_FTPR_NDIMS,
-    TRIXI_FTPR_NELEMENTS,
+    TRIXI_FPTR_NELEMENTS,
+    TRIXI_FPTR_NELEMENTS_GLOBAL,
     TRIXI_FTPR_NVARIABLES,
     TRIXI_FTPR_LOAD_CELL_AVERAGES,
     TRIXI_FTPR_VERSION_LIBRARY,
@@ -39,7 +40,8 @@ static const char* trixi_function_pointer_names[] = {
     [TRIXI_FTPR_STEP]                   = "trixi_step_cfptr",
     [TRIXI_FTPR_FINALIZE_SIMULATION]    = "trixi_finalize_simulation_cfptr",
     [TRIXI_FTPR_NDIMS]                  = "trixi_ndims_cfptr",
-    [TRIXI_FTPR_NELEMENTS]              = "trixi_nelements_cfptr",
+    [TRIXI_FPTR_NELEMENTS]              = "trixi_nelements_cfptr",
+    [TRIXI_FPTR_NELEMENTS_GLOBAL]       = "trixi_nelements_global_cfptr",
     [TRIXI_FTPR_NVARIABLES]             = "trixi_nvariables_cfptr",
     [TRIXI_FTPR_LOAD_CELL_AVERAGES]     = "trixi_load_cell_averages_cfptr",
     [TRIXI_FTPR_VERSION_LIBRARY]        = "trixi_version_library_cfptr",
@@ -443,20 +445,45 @@ int trixi_ndims(int handle) {
 /**
  * @anchor trixi_nelements_api_c
  *
- * @brief Return number of elements (cells)
+ * @brief Return number of local elements (cells).
+ *
+ * These usually differ from the global count when doing parallel computations.
  *
  * @param[in]  handle  simulation handle
+ *
+ * @see trixi_nelements_global_api_c
  */
 int trixi_nelements(int handle) {
 
     // Get function pointer
-    int (*nelements)(int) = trixi_function_pointers[TRIXI_FTPR_NELEMENTS];
+    int (*nelements)(int) = trixi_function_pointers[TRIXI_FPTR_NELEMENTS];
 
     // Call function
     return nelements(handle);
 }
 
 
+/**
+ * @anchor trixi_nelements_global_api_c
+ *
+ * @brief Return number of global elements (cells).
+ *
+ * These usually differ from the local count when doing parallel computations.
+ *
+ * @param[in]  handle  simulation handle
+ *
+ * @see trixi_nelements_api_c
+ */
+int trixi_nelements_global(int handle) {
+
+    // Get function pointer
+    int (*nelements_global)(int) = trixi_function_pointers[TRIXI_FPTR_NELEMENTS_GLOBAL];
+
+    // Call function
+    return nelements_global(handle);
+}
+
+
 /**
  * @anchor trixi_nvariables_api_c
  *

src/api.f90

@@ -248,7 +248,7 @@ integer(c_int) function trixi_ndims(handle) bind(c)
     !>
     !! @fn LibTrixi::trixi_nelements::trixi_nelements(handle)
     !!
-    !! @brief Return number of elements (cells)
+    !! @brief Return number of local elements (cells)
     !!
     !! @param[in]  handle  simulation handle
     !!
@@ -258,6 +258,19 @@ integer(c_int) function trixi_nelements(handle) bind(c)
       integer(c_int), value, intent(in) :: handle
     end function
 
+    !>
+    !! @fn LibTrixi::trixi_nelements_global::trixi_nelements_global(handle)
+    !!
+    !! @brief Return number of global elements (cells)
+    !!
+    !! @param[in]  handle  simulation handle
+    !!
+    !! @see @ref trixi_nelements_global_api_c "trixi_nelements_global (C API)"
+    integer(c_int) function trixi_nelements_global(handle) bind(c)
+      use, intrinsic :: iso_c_binding, only: c_int
+      integer(c_int), value, intent(in) :: handle
+    end function
+
     !>
     !! @fn LibTrixi::trixi_nvariables::trixi_nvariables(handle)
     !!

src/trixi.h

@@ -27,6 +27,7 @@ void trixi_step(int handle);
 // Simulation data
 int trixi_ndims(int handle);
 int trixi_nelements(int handle);
+int trixi_nelements_global(int handle);
 int trixi_nvariables(int handle);
 double trixi_calculate_dt(int handle);
 void trixi_load_cell_averages(double * data, int handle);

test/c/CMakeLists.txt

@@ -2,7 +2,8 @@ include( GoogleTest )
 
 set ( TESTS
       auxiliary.cpp
-      interface_c.cpp )
+      interface_c.cpp
+      simulation.cpp )
 
 
 foreach ( TEST ${TESTS} )
@@ -16,7 +17,7 @@ foreach ( TEST ${TESTS} )
     # set libraries to link
     target_link_libraries(
         ${TARGET_NAME}
-        PRIVATE MPI::MPI_C ${PROJECT_NAME} ${PROJECT_NAME}_tls GTest::gtest_main
+        PRIVATE MPI::MPI_CXX ${PROJECT_NAME} ${PROJECT_NAME}_tls GTest::gtest_main
     )
 
     # set include directories
@@ -36,3 +37,12 @@ foreach ( TEST ${TESTS} )
     gtest_discover_tests( ${TARGET_NAME} )
 
 endforeach()
+
+set_property(TARGET simulation
+             PROPERTY CROSSCOMPILING_EMULATOR
+             ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} 2
+            )
+
+# manually add MPI test
+gtest_add_tests( TARGET simulation
+                 TEST_SUFFIX "_MPI" )

test/c/interface_c.cpp

@@ -130,59 +130,3 @@ TEST(CInterfaceTest, FunctionPointers) {
     // Finalize libtrixi
     trixi_finalize();
 }
-
-
-TEST(CInterfaceTest, SimulationRun) {
-
-    // Initialize libtrixi
-    trixi_initialize( julia_project_path, NULL );
-
-    // Set up the Trixi simulation, get a handle
-    int handle = trixi_initialize_simulation(libelixir_path);
-    EXPECT_EQ(handle, 1);
-
-    // Using a non-existent handle should fail and exit
-    EXPECT_DEATH(trixi_is_finished(42),
-                 "the provided handle was not found in the stored simulation states: 42");
-
-    // Do a simulation step
-    trixi_step(handle);
-
-    // Check time step length
-    double dt = trixi_calculate_dt(handle);
-    EXPECT_DOUBLE_EQ(dt, 0.0032132984504400627);
-
-    // Check finished status
-    int finished_status = trixi_is_finished(handle);
-    EXPECT_EQ(finished_status, 0);
-
-    // Check number of dimensions
-    int ndims = trixi_ndims(handle);
-    EXPECT_EQ(ndims, 2);
-
-    // Check number of elements
-    int nelements = trixi_nelements(handle);
-    EXPECT_EQ(nelements, 256);
-
-    // Check number of variables
-    int nvariables = trixi_nvariables(handle);
-    EXPECT_EQ(nvariables, 4);
-
-    // Check cell averaged values
-    int size = nelements * nvariables;
-    std::vector<double> cell_averages(size);
-    trixi_load_cell_averages(cell_averages.data(), handle);
-    EXPECT_DOUBLE_EQ(cell_averages[0],      1.0);
-    EXPECT_DOUBLE_EQ(cell_averages[928],    2.6605289164377273);
-    EXPECT_DOUBLE_EQ(cell_averages[size-1], 1e-5);
-
-    // Finalize Trixi simulation
-    trixi_finalize_simulation( handle );
-
-    // Handle is now invalid and subsequent use should fail
-    EXPECT_DEATH(trixi_is_finished(handle),
-                 "the provided handle was not found in the stored simulation states: 1");
-
-    // Finalize libtrixi
-    trixi_finalize();
-}