Adding MPI parallelization to CFBM

Registry/Registry.EM_COMMON

@@ -3549,6 +3549,8 @@ halo      HALO_EM_COUPLE_B   dyn_em 48:ph_1,ph_2,w_1,w_2,t_1,t_2,u_1,u_2,v_1,v_2
 period    PERIOD_EM_COUPLE_B dyn_em 3:ph_1,ph_2,w_1,w_2,t_1,t_2,u_1,u_2,v_1,v_2,\
                                        moist,chem,tracer,scalar
 
+halo      HALO_EM_CFBM dyn_em 8:znt,t2,q2,psfc,rainc,rainnc,u10,v10
+
 #cyl for trajectory 
 halo      HALO_EM_F dyn_em  24:muu,muv,mut
 halo      HALO_EM_F_1 dyn_em  24:muus,muvs

doc/README.cfbm

@@ -0,0 +1,35 @@
+
+WRF should be compiled with CMake in order to use the Community Fire Behavior model:
+git clone --recurse-submodules https://github.com/wrf-model/WRF
+./configure_new -- -DENABLE_CFBM=ON
+./compile_new
+
+To activate the model you need to set ifire = 1 in the fire block of the WRF namelist.
+This instructs WRF to read the CFBM namelist: namelist.cfbm
+
+In a given WRF simulation, CFBM can only be activated in one domain.
+
+It only works with Lambert-Conformal projections at this time.
+
+This is an example of the namelist.cfbm:
+
+ &time
+ dt = 2               ! WRF time step [s] for the nest CFBM is active
+ interval_output = 2  ! Frequency [s] saving fire output
+/
+
+ &atm
+ /
+
+ &fire
+ fire_num_ignitions = 1,
+ fire_ignition_ros1 =   1.2,
+ fire_ignition_start_lat1 = 39.68,
+ fire_ignition_start_lon1 = -103.58,
+ fire_ignition_end_lat1 = 39.68,
+ fire_ignition_end_lon1 = -103.58,
+ fire_ignition_radius1 = 2000.0,
+ fire_ignition_start_time1 = 6.0,
+ fire_ignition_end_time1 = 60.0,
+/
+

dyn_em/module_first_rk_step_part1.F

@@ -56,18 +56,19 @@ SUBROUTINE first_rk_step_part1 (   grid , config_flags              &
 #ifdef DM_PARALLEL
     USE module_dm, ONLY : local_communicator, mytask, ntasks, ntasks_x, ntasks_y, local_communicator_periodic, wrf_dm_maxval
     USE module_comm_dm, ONLY : halo_em_phys_a_sub,halo_em_fdda_sfc_sub,halo_pwp_sub,halo_em_chem_e_3_sub, &
-    halo_em_chem_e_5_sub, halo_em_hydro_noahmp_sub
+    halo_em_chem_e_5_sub, halo_em_hydro_noahmp_sub, halo_em_tracer_e_5_sub, halo_em_cfbm_sub
 #if ( WRFPLUS == 1 )
     USE module_comm_dm, ONLY : halo_em_phys_a_bl_surf_sub
 #endif
 #endif
     USE module_utility
 
 #ifdef WRF_CFBM
-      ! CFBM
+      ! CFBM modules
     use advance_mod, only : Advance_state
-    use wrf_mod, only : Interp_wrf2dvar_to_cfbm, Interp_wrfwinds_to_cfbm
-    use wrf_mod, only : Provide_atm_feedback
+    use wrf_mod, only : Interp_wrfwinds_to_cfbm, Provide_atm_feedback
+    use interp_mod, only : VINTERP_WINDS_FROM_3D_WINDS, VINTERP_WINDS_FROM_10M_WINDS
+    use coupling_mod, only : Interp_horizontal
 #endif
 
     IMPLICIT NONE
@@ -1408,38 +1409,68 @@ SUBROUTINE first_rk_step_part1 (   grid , config_flags              &
 #ifdef WRF_CFBM
        case (1)
            ! Interpolate from Atm to fire grid
-         call Interp_wrf2dvar_to_cfbm (grid%znt, ims, ime, jms, jme, &
-             grid%fire_state%ifms, grid%fire_state%ifme, grid%fire_state%jfms, grid%fire_state%jfme, &
-             grid%fire_state%ifps, grid%fire_state%ifpe, grid%fire_state%jfps, grid%fire_state%jfpe, &
-             grid%fire_state%lats, grid%fire_state%lons, grid%fire_state%proj, grid%fire_state%fz0)
-
-         call Interp_wrf2dvar_to_cfbm (grid%t2, ims, ime, jms, jme, &
+#ifdef DM_PARALLEL
+#     include "HALO_EM_CFBM.inc"
+#endif
+         call Interp_horizontal (grid%znt, grid%fire_state%proj, ims, ime, jms, jme, &
              grid%fire_state%ifms, grid%fire_state%ifme, grid%fire_state%jfms, grid%fire_state%jfme, &
-             grid%fire_state%ifps, grid%fire_state%ifpe, grid%fire_state%jfps, grid%fire_state%jfpe, &
-             grid%fire_state%lats, grid%fire_state%lons, grid%fire_state%proj, grid%fire_state%fire_t2)
+             grid%fire_state%num_tiles, grid%fire_state%i_start, grid%fire_state%i_end, grid%fire_state%j_start, &
+             grid%fire_state%j_end, grid%cfbm_namelist%hinterp_opt, grid%fire_state%lats, grid%fire_state%lons, &
+             grid%fire_state%fz0)
 
-         call Interp_wrf2dvar_to_cfbm (grid%q2, ims, ime, jms, jme, &
+         call Interp_horizontal (grid%t2, grid%fire_state%proj, ims, ime, jms, jme, &
              grid%fire_state%ifms, grid%fire_state%ifme, grid%fire_state%jfms, grid%fire_state%jfme, &
-             grid%fire_state%ifps, grid%fire_state%ifpe, grid%fire_state%jfps, grid%fire_state%jfpe, &
-             grid%fire_state%lats, grid%fire_state%lons, grid%fire_state%proj, grid%fire_state%fire_q2)
+             grid%fire_state%num_tiles, grid%fire_state%i_start, grid%fire_state%i_end, grid%fire_state%j_start, &
+             grid%fire_state%j_end, grid%cfbm_namelist%hinterp_opt, grid%fire_state%lats, grid%fire_state%lons, &
+             grid%fire_state%fire_t2)
 
-         call Interp_wrf2dvar_to_cfbm (grid%psfc, ims, ime, jms, jme, &
+         call Interp_horizontal (grid%q2, grid%fire_state%proj, ims, ime, jms, jme, &
              grid%fire_state%ifms, grid%fire_state%ifme, grid%fire_state%jfms, grid%fire_state%jfme, &
-             grid%fire_state%ifps, grid%fire_state%ifpe, grid%fire_state%jfps, grid%fire_state%jfpe, &
-             grid%fire_state%lats, grid%fire_state%lons, grid%fire_state%proj, grid%fire_state%fire_psfc)
+             grid%fire_state%num_tiles, grid%fire_state%i_start, grid%fire_state%i_end, grid%fire_state%j_start, &
+             grid%fire_state%j_end, grid%cfbm_namelist%hinterp_opt, grid%fire_state%lats, grid%fire_state%lons, &
+             grid%fire_state%fire_q2)
 
-         call Interp_wrf2dvar_to_cfbm (grid%rainc + grid%rainnc, ims, ime, jms, jme, &
+         call Interp_horizontal (grid%psfc, grid%fire_state%proj, ims, ime, jms, jme, &
              grid%fire_state%ifms, grid%fire_state%ifme, grid%fire_state%jfms, grid%fire_state%jfme, &
-             grid%fire_state%ifps, grid%fire_state%ifpe, grid%fire_state%jfps, grid%fire_state%jfpe, &
-             grid%fire_state%lats, grid%fire_state%lons, grid%fire_state%proj, grid%fire_state%fire_rain)
+             grid%fire_state%num_tiles, grid%fire_state%i_start, grid%fire_state%i_end, grid%fire_state%j_start, &
+             grid%fire_state%j_end, grid%cfbm_namelist%hinterp_opt, grid%fire_state%lats, grid%fire_state%lons, &
+             grid%fire_state%fire_psfc)
 
-         call Interp_wrfwinds_to_cfbm (grid%u_phy, grid%v_phy, grid%z_at_w, ims, ime, kms, kme, jms, jme, &
+         call Interp_horizontal (grid%rainc + grid%rainnc, grid%fire_state%proj, ims, ime, jms, jme, &
              grid%fire_state%ifms, grid%fire_state%ifme, grid%fire_state%jfms, grid%fire_state%jfme, &
-             grid%fire_state%ifps, grid%fire_state%ifpe, grid%fire_state%jfps, grid%fire_state%jfpe, &
-             grid%fire_state%kfds, grid%fire_state%kfde, &
-             grid%fire_state%lats, grid%fire_state%lons, grid%fire_state%proj, grid%fire_state%fz0,  &
-             grid%cfbm_namelist%fire_lsm_zcoupling, grid%cfbm_namelist%fire_lsm_zcoupling_ref, &
-             grid%cfbm_namelist%fire_wind_height, grid%fire_state%uf, grid%fire_state%vf)
+             grid%fire_state%num_tiles, grid%fire_state%i_start, grid%fire_state%i_end, grid%fire_state%j_start, &
+             grid%fire_state%j_end, grid%cfbm_namelist%hinterp_opt, grid%fire_state%lats, grid%fire_state%lons, &
+             grid%fire_state%fire_rain)
+
+         Select_vinterp_opt: select case  (grid%cfbm_namelist%wind_vinterp_opt)
+           case (VINTERP_WINDS_FROM_3D_WINDS)
+             call Interp_wrfwinds_to_cfbm (grid%u_phy, grid%v_phy, grid%z_at_w, ims, ime, kms, kme, jms, jme, &
+                 grid%fire_state%ifms, grid%fire_state%ifme, grid%fire_state%jfms, grid%fire_state%jfme, &
+                 grid%fire_state%ifps, grid%fire_state%ifpe, grid%fire_state%jfps, grid%fire_state%jfpe, &
+                 grid%fire_state%kfds, grid%fire_state%kfde, &
+                 grid%fire_state%lats, grid%fire_state%lons, grid%fire_state%proj, grid%fire_state%fz0,  &
+                 grid%cfbm_namelist%fire_lsm_zcoupling, grid%cfbm_namelist%fire_lsm_zcoupling_ref, &
+                 grid%cfbm_namelist%fire_wind_height, grid%fire_state%uf, grid%fire_state%vf)
+
+           case (VINTERP_WINDS_FROM_10M_WINDS)
+             call Interp_horizontal (grid%u10, grid%fire_state%proj, ims, ime, jms, jme, &
+                 grid%fire_state%ifms, grid%fire_state%ifme, grid%fire_state%jfms, grid%fire_state%jfme, &
+                 grid%fire_state%num_tiles, grid%fire_state%i_start, grid%fire_state%i_end, grid%fire_state%j_start, &
+                 grid%fire_state%j_end, grid%cfbm_namelist%hinterp_opt, grid%fire_state%lats, grid%fire_state%lons, &
+                 grid%fire_state%uf)
+
+             call Interp_horizontal (grid%v10, grid%fire_state%proj, ims, ime, jms, jme, &
+                 grid%fire_state%ifms, grid%fire_state%ifme, grid%fire_state%jfms, grid%fire_state%jfme, &
+                 grid%fire_state%num_tiles, grid%fire_state%i_start, grid%fire_state%i_end, grid%fire_state%j_start, &
+                 grid%fire_state%j_end, grid%cfbm_namelist%hinterp_opt, grid%fire_state%lats, grid%fire_state%lons, &
+                 grid%fire_state%vf)
+
+             call grid%fire_state%Apply_wafs ()
+
+           case default
+             call wrf_error_fatal ('The vertical wind interpolation option does not exist in CFBM')
+
+         end select Select_vinterp_opt
 
          call Advance_state (grid = grid%fire_state, config_flags = grid%cfbm_namelist)
 
@@ -1475,6 +1506,12 @@ SUBROUTINE first_rk_step_part1 (   grid , config_flags              &
                grid%rthfrten, grid%rqvfrten)          ! theta and Qv tendencies
            end do
            !$OMP END PARALLEL DO
+
+           if (config_flags%tracer_opt == 3) then
+#ifdef DM_PARALLEL
+#     include "HALO_EM_TRACER_E_5.inc"
+#endif
+           end if
          else
            call wrf_error_fatal ('num_tiles in WRF and CFBM should match')
          endif

dyn_em/start_em.F

@@ -20,11 +20,12 @@ SUBROUTINE start_domain_em ( grid, allowed_to_read &
    USE module_dm, ONLY : wrf_dm_min_real, wrf_dm_max_real, wrf_dm_maxval, &
         ntasks_x, ntasks_y, &
         local_communicator_periodic, local_communicator, mytask, ntasks
+
 #else
    USE module_dm, ONLY : wrf_dm_min_real, wrf_dm_max_real
 #endif
    USE module_comm_dm
-   USE module_llxy, ONLY : proj_cassini
+   USE module_llxy, ONLY : proj_cassini, PROJ_LC
    USE module_physics_init
    USE NoahmpGroundwaterInitMod, ONLY: NoahmpGroundwaterInitMain
    USE module_lightning_driver, ONLY : lightning_init
@@ -145,6 +146,7 @@ SUBROUTINE start_domain_em ( grid, allowed_to_read &
               ifms, ifme, jfms, jfme, kfms, kfme, &
               ifps, ifpe, jfps, jfpe, kfps, kfpe
    real :: cen_lat, cen_lon, truelat1, truelat2, stand_lon
+   logical :: is_cfbm_initialized = .false.
 #endif
 
 
@@ -2246,6 +2248,20 @@ SUBROUTINE start_domain_em ( grid, allowed_to_read &
        ! Community Fire Behavior model
      case (1)
 #ifdef WRF_CFBM
+       Init_cfbm: if (.not. is_cfbm_initialized) then
+#ifdef DM_PARALLEL
+         ! Broadcast the namelist
+       call grid%cfbm_namelist%Broadcast_nml (grid%communicator)
+#endif
+
+       if (abs (grid%dt - grid%cfbm_namelist%dt) > 1.0e-5) then
+         call wrf_error_fatal ('CFBM dt should match WRF dt for the fire domain')
+       end if
+
+       if (config_flags%map_proj /= PROJ_LC) then
+         call wrf_error_fatal ('CFBM only supports Lambert Conformal projections at this moment')
+       end if
+
          ! Get fire dims
        call get_ijk_from_subgrid (grid, ifds, ifde, jfds, jfde, kfds, kfde, &
                                         ifms, ifme, jfms, jfme, kfms, kfme, &
@@ -2280,7 +2296,11 @@ SUBROUTINE start_domain_em ( grid, allowed_to_read &
            sr_x = config_flags%sr_x, sr_y = config_flags%sr_y, &
            map_proj = config_flags%map_proj, cen_lat = cen_lat, cen_lon = cen_lon, truelat1 = truelat1, &
            truelat2 = truelat2, stand_lon = stand_lon, nfuel_cat = grid%nfuel_cat, zsf = grid%zsf, &
-           dzdxf = grid%dzdxf, dzdyf = grid%dzdyf)
+           dzdxf = grid%dzdxf, dzdyf = grid%dzdyf, communicator = grid%communicator)
+
+         is_cfbm_initialized = .true.
+
+       end if Init_cfbm
 #else
        call wrf_error_fatal ('CFBM requires compilation with CMake')
 #endif

share/module_check_a_mundo.F

@@ -2774,6 +2774,24 @@ END FUNCTION bep_bem_ngr_u
         count_fatal_error = count_fatal_error + 1
       END IF
 
+!-----------------------------------------------------------------------
+! CFBM checks
+!-----------------------------------------------------------------------
+
+      ! CFBM can be active in only one domain
+      oops = 0
+      DO i = 1, model_config_rec % max_dom
+        IF (model_config_rec%ifire(i) == 1) THEN
+          oops = oops + 1
+        END IF
+      END DO
+
+      IF ( oops .GT. 1 ) THEN
+        wrf_err_message = '--- ERROR: CFBM can be active in only one domain'
+        CALL wrf_debug ( 0, TRIM( wrf_err_message ) )
+        count_fatal_error = count_fatal_error + 1
+      END IF
+
 !-----------------------------------------------------------------------
 ! This WRF version does not support trajectories on a global domain
 !-----------------------------------------------------------------------