Perftest: Add device memory with DMA-BUF export support

Introduce a new memory backend using ibv_alloc_dm() and
ibv_dm_export_dmabuf_fd() to allocate and export device
memory as DMA-BUF file descriptors for RDMA testing.

Usage: --use_ib_dm_dmabuf=<ib_device>

Also guard dmabuf registration prints behind verbosity level
so --output=bandwidth produces clean output for all backends.

Not supported with write latency tests.

Signed-off-by: Shmuel Shaul <sshaul@nvidia.com>

Author: sshaulnv

Makefile.am

@@ -35,7 +35,7 @@ AUTOMAKE_OPTIONS= subdir-objects
 
 noinst_LIBRARIES = libperftest.a
 libperftest_a_SOURCES = src/get_clock.c src/perftest_communication.c src/perftest_parameters.c src/perftest_resources.c src/perftest_counters.c src/host_memory.c src/mmap_memory.c
-noinst_HEADERS = src/get_clock.h src/perftest_communication.h src/perftest_parameters.h src/perftest_resources.h src/perftest_counters.h src/memory.h src/host_memory.h src/mmap_memory.h src/cuda_memory.h src/rocm_memory.h src/neuron_memory.h src/hl_memory.h src/mlu_memory.h
+noinst_HEADERS = src/get_clock.h src/perftest_communication.h src/perftest_parameters.h src/perftest_resources.h src/perftest_counters.h src/memory.h src/host_memory.h src/mmap_memory.h src/cuda_memory.h src/rocm_memory.h src/neuron_memory.h src/hl_memory.h src/mlu_memory.h src/dm_memory.h
 
 if CUDA
 libperftest_a_SOURCES += src/cuda_memory.c src/cuda_loader.h src/cuda_loader.c
@@ -61,6 +61,10 @@ if OPENCL
 libperftest_a_SOURCES += src/opencl_memory.c
 endif
 
+if IB_DM_DMABUF
+libperftest_a_SOURCES += src/dm_memory.c
+endif
+
 bin_PROGRAMS = ib_send_bw ib_send_lat ib_write_lat ib_write_bw ib_read_lat ib_read_bw ib_atomic_lat ib_atomic_bw
 bin_SCRIPTS = run_perftest_loopback run_perftest_multi_devices
 

configure.ac

@@ -77,6 +77,12 @@ if test $HAVE_REG_DMABUF_MR = yes; then
 	AC_DEFINE([HAVE_REG_DMABUF_MR], [1], [Enable HAVE_REG_DMABUF_MR])
 fi
 
+AC_CHECK_LIB([ibverbs], [ibv_dm_export_dmabuf_fd], [HAVE_IB_DM_DMABUF=yes], [HAVE_IB_DM_DMABUF=no])
+AM_CONDITIONAL([IB_DM_DMABUF], [test "x$HAVE_IB_DM_DMABUF" = "xyes"])
+if test "x$HAVE_IB_DM_DMABUF" = "xyes"; then
+	AC_DEFINE([HAVE_IB_DM_DMABUF], [1], [Enable IB DM DMABUF export])
+fi
+
 AC_TRY_LINK([#include <infiniband/verbs.h>],
 	[struct ibv_flow *t = ibv_create_flow(NULL,NULL);],[HAVE_RAW_ETH_REG=yes], [HAVE_RAW_ETH_REG=no])
 

man/perftest.1

@@ -402,6 +402,13 @@ many different options and modes.
  Use MLU DMA-BUF for HW accelerator direct RDMA testing.
  System support required.
 .TP
+.B --use_ib_dm_dmabuf=<ib_device>
+ Use IB device memory (allocated via ibv_alloc_dm) with DMA-BUF export
+ (via ibv_dm_export_dmabuf_fd) for RDMA testing. The <ib_device> argument
+ specifies the IB device name (e.g. mlx5_0). The requested message size
+ must fit within the device's max_dm_size.
+ System support required (rdma-core with ibv_dm_export_dmabuf_fd).
+.TP
 .B --use_opencl=<opencl device id>
  Use OpenCl specific device for GPUDirect RDMA testing
  Not relevant for raw_ethernet_fs_rate.

src/dm_memory.c

@@ -0,0 +1,184 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <unistd.h>
+#include <infiniband/verbs.h>
+#include "dm_memory.h"
+#include "perftest_parameters.h"
+
+struct dm_memory_ctx {
+	struct memory_ctx base;
+	char *ib_devname;
+	struct ibv_context *ib_ctx;
+	struct ibv_dm *dm;
+	uint64_t dm_size;
+	bool use_dmabuf;
+	enum verbosity_level output;
+};
+
+static int dm_memory_init(struct memory_ctx *ctx)
+{
+	struct dm_memory_ctx *dm_ctx = container_of(ctx, struct dm_memory_ctx, base);
+	struct ibv_device **dev_list;
+	struct ibv_device *ib_dev = NULL;
+	int num_devices;
+	int i;
+
+	dev_list = ibv_get_device_list(&num_devices);
+	if (!dev_list) {
+		fprintf(stderr, "Failed to get IB devices list\n");
+		return FAILURE;
+	}
+
+	for (i = 0; i < num_devices; i++) {
+		if (!strcmp(ibv_get_device_name(dev_list[i]), dm_ctx->ib_devname)) {
+			ib_dev = dev_list[i];
+			break;
+		}
+	}
+
+	if (!ib_dev) {
+		fprintf(stderr, "IB device %s not found\n", dm_ctx->ib_devname);
+		ibv_free_device_list(dev_list);
+		return FAILURE;
+	}
+
+	dm_ctx->ib_ctx = ibv_open_device(ib_dev);
+	ibv_free_device_list(dev_list);
+
+	if (!dm_ctx->ib_ctx) {
+		fprintf(stderr, "Couldn't open IB device %s\n", dm_ctx->ib_devname);
+		return FAILURE;
+	}
+
+	if (dm_ctx->output == FULL_VERBOSITY)
+		printf("Opened IB device %s for device memory\n", dm_ctx->ib_devname);
+	return SUCCESS;
+}
+
+static int dm_memory_destroy(struct memory_ctx *ctx)
+{
+	struct dm_memory_ctx *dm_ctx = container_of(ctx, struct dm_memory_ctx, base);
+
+	if (dm_ctx->ib_ctx)
+		ibv_close_device(dm_ctx->ib_ctx);
+
+	free(dm_ctx);
+	return SUCCESS;
+}
+
+static int dm_memory_allocate_buffer(struct memory_ctx *ctx, int alignment,
+				     uint64_t size, int *dmabuf_fd,
+				     uint64_t *dmabuf_offset, void **addr,
+				     bool *can_init)
+{
+	struct dm_memory_ctx *dm_ctx = container_of(ctx, struct dm_memory_ctx, base);
+	struct ibv_alloc_dm_attr dm_attr = {};
+	struct ibv_device_attr_ex dev_attr = {};
+	int ret;
+
+	ret = ibv_query_device_ex(dm_ctx->ib_ctx, NULL, &dev_attr);
+	if (ret) {
+		fprintf(stderr, "Failed to query device attributes: %d\n", ret);
+		return FAILURE;
+	}
+
+	if (size > dev_attr.max_dm_size) {
+		fprintf(stderr, "Requested DM size %lu exceeds device max %lu\n",
+			(unsigned long)size, (unsigned long)dev_attr.max_dm_size);
+		return FAILURE;
+	}
+
+	dm_attr.length = size;
+
+	dm_ctx->dm = ibv_alloc_dm(dm_ctx->ib_ctx, &dm_attr);
+	if (!dm_ctx->dm) {
+		fprintf(stderr, "Failed to allocate device memory of size %lu: %s\n",
+			(unsigned long)size, strerror(errno));
+		return FAILURE;
+	}
+
+	dm_ctx->dm_size = size;
+	*addr = NULL;
+	*can_init = false;
+
+	if (dm_ctx->output == FULL_VERBOSITY)
+		printf("Allocated %lu bytes of device memory\n", (unsigned long)size);
+
+	if (dm_ctx->use_dmabuf) {
+		int fd = ibv_dm_export_dmabuf_fd(dm_ctx->dm);
+		if (fd < 0) {
+			fprintf(stderr, "Failed to export DM as dmabuf fd: %s\n",
+				strerror(errno));
+			ibv_free_dm(dm_ctx->dm);
+			dm_ctx->dm = NULL;
+			return FAILURE;
+		}
+
+		*dmabuf_fd = fd;
+		*dmabuf_offset = 0;
+		if (dm_ctx->output == FULL_VERBOSITY)
+			printf("Exported DM as dmabuf fd %d\n", fd);
+	}
+
+	return SUCCESS;
+}
+
+static int dm_memory_free_buffer(struct memory_ctx *ctx, int dmabuf_fd,
+				 void *addr, uint64_t size)
+{
+	struct dm_memory_ctx *dm_ctx = container_of(ctx, struct dm_memory_ctx, base);
+
+	if (dm_ctx->dm) {
+		ibv_free_dm(dm_ctx->dm);
+		dm_ctx->dm = NULL;
+	}
+
+	return SUCCESS;
+}
+
+static void *dm_memory_copy_host_to_buffer(void *dest, const void *src, size_t size)
+{
+	/*
+	 * DM buffers don't have a direct host address. The actual data path
+	 * goes through RDMA operations, so host-side copy is a no-op.
+	 */
+	return dest;
+}
+
+static void *dm_memory_copy_buffer_to_host(void *dest, const void *src, size_t size)
+{
+	return dest;
+}
+
+static void *dm_memory_copy_buffer_to_buffer(void *dest, const void *src, size_t size)
+{
+	return dest;
+}
+
+bool dm_memory_dmabuf_supported()
+{
+	return true;
+}
+
+struct memory_ctx *dm_memory_create(struct perftest_parameters *params)
+{
+	struct dm_memory_ctx *ctx;
+
+	ALLOCATE(ctx, struct dm_memory_ctx, 1);
+	ctx->base.init = dm_memory_init;
+	ctx->base.destroy = dm_memory_destroy;
+	ctx->base.allocate_buffer = dm_memory_allocate_buffer;
+	ctx->base.free_buffer = dm_memory_free_buffer;
+	ctx->base.copy_host_to_buffer = dm_memory_copy_host_to_buffer;
+	ctx->base.copy_buffer_to_host = dm_memory_copy_buffer_to_host;
+	ctx->base.copy_buffer_to_buffer = dm_memory_copy_buffer_to_buffer;
+	ctx->ib_devname = params->dm_ib_devname;
+	ctx->use_dmabuf = params->use_ib_dm_dmabuf;
+	ctx->output = params->output;
+	ctx->dm = NULL;
+	ctx->ib_ctx = NULL;
+
+	return &ctx->base;
+}

src/dm_memory.h

@@ -0,0 +1,27 @@
+#ifndef DM_MEMORY_H
+#define DM_MEMORY_H
+
+#include "memory.h"
+#include "config.h"
+
+
+struct perftest_parameters;
+
+bool dm_memory_dmabuf_supported();
+
+struct memory_ctx *dm_memory_create(struct perftest_parameters *params);
+
+
+#ifndef HAVE_IB_DM_DMABUF
+
+inline bool dm_memory_dmabuf_supported() {
+	return false;
+}
+
+inline struct memory_ctx *dm_memory_create(struct perftest_parameters *params) {
+	return NULL;
+}
+
+#endif
+
+#endif /* DM_MEMORY_H */

src/perftest_communication.c

@@ -1033,7 +1033,8 @@ int negotiate_params(struct pingpong_context *ctx,
 	};
 	#undef COMPARE
 
-	for (int i = 0; i < sizeof(compare_functions)/sizeof(compare_functions[0]); i++) {
+	int i;
+	for (i = 0; i < sizeof(compare_functions)/sizeof(compare_functions[0]); i++) {
 		if (compare_functions[i].compare_func(compare_functions[i].name,
 											  compare_functions[i].local_value,
 											  compare_functions[i].remote_value,

src/perftest_parameters.c

@@ -20,6 +20,7 @@
 #include "hl_memory.h"
 #include "mlu_memory.h"
 #include "opencl_memory.h"
+#include "dm_memory.h"
 #include<math.h>
 #ifdef HAVE_RO
 #include <stdbool.h>
@@ -700,6 +701,11 @@ static void usage(const char *argv0, VerbType verb, TestType tst, int connection
 			printf(" Use OpenCl specific platform ID\n");
 		}
 
+		if (dm_memory_dmabuf_supported()) {
+			printf("      --use_ib_dm_dmabuf=<ib_device>");
+			printf(" Use device memory with DMA-BUF export for RDMA testing\n");
+		}
+
 		if (cuda_memory_supported() ||
 		    opencl_memory_supported()) {
 			printf("      --gpu_touch=<once\\infinite> ");
@@ -947,6 +953,8 @@ static void init_perftest_params(struct perftest_parameters *user_param)
 	user_param->use_mlu_dmabuf	= 0;
 	user_param->opencl_platform_id	= 0;
 	user_param->opencl_device_id	= 0;
+	user_param->dm_ib_devname	= NULL;
+	user_param->use_ib_dm_dmabuf	= 0;
 	user_param->gpu_touch		= GPU_NO_TOUCH;
 	user_param->mmap_file		= NULL;
 	user_param->mmap_offset		= 0;
@@ -1987,9 +1995,10 @@ static void force_dependecies(struct perftest_parameters *user_param)
 		exit(1);
 	}
 
-	if (user_param->memory_type == MEMORY_CUDA && user_param->verb == SEND && (user_param->size <= 64 || user_param->test_method == RUN_ALL)) {
+	if ((user_param->memory_type == MEMORY_CUDA || user_param->memory_type == MEMORY_DM) &&
+	    user_param->verb == SEND && (user_param->size <= 64 || user_param->test_method == RUN_ALL)) {
 		printf(RESULT_LINE);
-		printf("Scatter2CQE (size <= 64) is not supported with GPUDirect send tests: setting MLX5_SCATTER_TO_CQE=0\n");
+		printf("Scatter2CQE (size <= 64) is not supported with device memory send tests: setting MLX5_SCATTER_TO_CQE=0\n");
 		if (disable_mlx5_scatter_to_cqe())
 			exit(1);
 	}
@@ -2012,6 +2021,12 @@ static void force_dependecies(struct perftest_parameters *user_param)
 		exit(1);
 	}
 
+	if (user_param->memory_type == MEMORY_DM && user_param->tst == LAT && (user_param->verb == WRITE || user_param->verb == WRITE_IMM)) {
+		printf(RESULT_LINE);
+		fprintf(stderr,"Perftest doesn't support device memory latency test with write verb\n");
+		exit(1);
+	}
+
 	if (user_param->use_data_direct) {
 		user_param->use_cuda_pcie_mapping = 1;
 	}
@@ -2517,6 +2532,11 @@ static void ctx_set_max_inline(struct ibv_context *context,struct perftest_param
 			return;
 		}
 
+		if (user_param->memory_type == MEMORY_DM){
+			user_param->inline_size = 0;
+			return;
+		}
+
 		if (user_param->tst == LAT) {
 			switch(user_param->verb) {
 				case WRITE_IMM:
@@ -2633,6 +2653,7 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 	static int use_mlu_dmabuf_flag = 0;
 	static int use_opencl_flag = 0;
 	static int opencl_platform_id_flag = 0;
+	static int use_ib_dm_dmabuf_flag = 0;
 	static int gpu_touch_flag = 0;
 	static int disable_pcir_flag = 0;
 	static int mmap_file_flag = 0;
@@ -2825,6 +2846,7 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 			{ .name = "use_mlu_dmabuf",	.has_arg = 0, .flag = &use_mlu_dmabuf_flag, .val = 1},
 			{ .name = "use_opencl",         .has_arg = 1, .flag = &use_opencl_flag, .val = 1},
 			{ .name = "opencl_platform_id", .has_arg = 1, .flag = &opencl_platform_id_flag, .val = 1},
+			{ .name = "use_ib_dm_dmabuf",	.has_arg = 1, .flag = &use_ib_dm_dmabuf_flag, .val = 1},
 			{ .name = "gpu_touch",		.has_arg = 1, .flag = &gpu_touch_flag, .val = 1},
 			{ .name = "mmap",		.has_arg = 1, .flag = &mmap_file_flag, .val = 1},
 			{ .name = "mmap-offset",	.has_arg = 1, .flag = &mmap_offset_flag, .val = 1},
@@ -3278,7 +3300,8 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 				    (use_hl_flag && !hl_memory_supported()) ||
 				    (use_mlu_flag && !mlu_memory_supported()) ||
 				    (use_mlu_dmabuf_flag && !mlu_memory_dmabuf_supported()) ||
-				    (use_opencl_flag && !opencl_memory_supported())) {
+				    (use_opencl_flag && !opencl_memory_supported()) ||
+				    (use_ib_dm_dmabuf_flag && !dm_memory_dmabuf_supported())) {
 					printf(" Unsupported memory type\n");
 					return FAILURE;
 				}
@@ -3289,6 +3312,7 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 				/* Memory types are mutually exclucive, make sure we were not already asked to use a different memory type. */
 				if (user_param->memory_type != MEMORY_HOST &&
 				    (mmap_file_flag || use_mlu_flag || use_neuron_flag || use_hl_flag ||
+						use_ib_dm_dmabuf_flag ||
 					 (use_rocm_flag && user_param->memory_type != MEMORY_ROCM) ||
 				     ((use_cuda_flag || use_cuda_bus_id_flag) && user_param->memory_type != MEMORY_CUDA))) {
 					fprintf(stderr, " Can't use multiple memory types\n");
@@ -3422,6 +3446,13 @@ int parser(struct perftest_parameters *user_param,char *argv[], int argc)
 					}
 					opencl_platform_id_flag = 0;
 				}
+				if (use_ib_dm_dmabuf_flag) {
+					user_param->dm_ib_devname = strdup(optarg);
+					user_param->use_ib_dm_dmabuf = 1;
+					user_param->memory_type = MEMORY_DM;
+					user_param->memory_create = dm_memory_create;
+					use_ib_dm_dmabuf_flag = 0;
+				}
 				if (gpu_touch_flag) {
 					if (!cuda_gpu_touch_supported()) {
 						fprintf(stderr, "GPU touch is not supported\n");