prov/psm3: Add missing ifdefs for rv module

Signed-off-by: Goldman, Adam <adam.goldman@intel.com>

Author: acgoldma

prov/psm3/psm3/psm_ep.c

@@ -991,15 +991,27 @@ __psm2_ep_open_internal(psm2_uuid_t const unique_job_key, int *devid_enabled,
 			// mode 2 (user space MR w/cache) is purposely not documented
 			psmi_getenv("PSM3_MR_CACHE_MODE",
 					"Enable MR caching 0=user space MR no cache"
+#ifdef RNDV_MOD_MR
 					", 1=kernel MR w/cache [1]",
+#else
+					"[0]",
+#endif
 					PSMI_ENVVAR_LEVEL_USER,
 					PSMI_ENVVAR_TYPE_UINT,
+#ifdef RNDV_MOD_MR
 					(union psmi_envvar_val)MR_CACHE_MODE_KERNEL,
+#else
+					(union psmi_envvar_val)MR_CACHE_MODE_NONE,
+#endif
 					 &env_mr_cache_mode);
 			if (! MR_CACHE_MODE_VALID(env_mr_cache_mode.e_uint)
 				|| env_mr_cache_mode.e_uint == MR_CACHE_MODE_RV)
 				env_mr_cache_mode.e_uint = MR_CACHE_MODE_NONE;
 		}
+#ifndef RNDV_MOD_MR
+		if (env_mr_cache_mode.e_uint == MR_CACHE_MODE_KERNEL)
+			env_mr_cache_mode.e_uint = MR_CACHE_MODE_NONE;
+#endif
 		ep->mr_cache_mode = env_mr_cache_mode.e_uint;
 	}
 

prov/psm3/psm3/psm_hal_gen1/psm_gdrcpy.c

@@ -94,7 +94,7 @@ gdr_convert_gpu_to_host_addr(int gdr_fd, unsigned long buf,
 
 	_HFI_VDBG("buf=%p size=%zu pageaddr=%p pagelen=%u flags=0x%x proto=%p\n",
 		(void *)buf, size, (void *)pageaddr, pagelen, flags, proto);
-
+#ifdef RNDV_MOD_MR
 	host_addr_buf = __psm2_rv_pin_and_mmap(proto->ep->verbs_ep.rv, pageaddr, pagelen);
 	if (! host_addr_buf) {
 		if (errno == ENOMEM || errno == EINVAL) {
@@ -110,6 +110,10 @@ gdr_convert_gpu_to_host_addr(int gdr_fd, unsigned long buf,
 			return NULL;
 		}
 	}
+#else
+	psmi_assert_always(0);	// unimplemented, should not get here
+	host_addr_buf = NULL;
+#endif /* RNDV_MOD_MR */
 	return host_addr_buf + (buf & GPU_PAGE_OFFSET_MASK);
 }
 
@@ -118,6 +122,7 @@ int
 gdr_unmap_gpu_host_addr(int gdr_fd, const void *buf,
 							 size_t size, struct ips_proto* proto)
 {
+#ifdef RNDV_MOD_MR
 	// TBD - will we need to round size up to pagelen?
 	if (0 != __psm2_rv_munmap_and_unpin(proto->ep->verbs_ep.rv, buf, size)) {
 		/* Fatal error */
@@ -127,6 +132,11 @@ gdr_unmap_gpu_host_addr(int gdr_fd, const void *buf,
 		return -1;
 	}
 	return 0;
+#else
+	psmi_assert_always(0);	// unimplemented, should not get here
+	errno = EINVAL;
+	return -1;
+#endif
 }
 
 

prov/psm3/psm3/ptl_ips/ips_expected_proto.h

@@ -201,10 +201,14 @@ struct ips_tid_send_desc {
 
 
 	uint8_t is_complete:1;	// all packets for send queued, waiting CQE/response
+#ifdef RNDV_MOD_MR
 	uint8_t rv_need_err_chk_rdma:1; // need to determine if a retry is required
 	uint8_t reserved:6;
 	uint8_t rv_sconn_index;	// sconn in rv we issued RDMA write on
 	uint32_t rv_conn_count;// Count of sconn completed conn establishments
+#else
+	uint8_t reserved:7;
+#endif
 
 #ifdef PSM_CUDA
 	/* As size of cuda_hostbuf is less than equal to window size,