Skip to content

Commit 5453511

Browse files
dbatheeshsshaulnv
dbatheesh
authored and
sshaulnv
committed
Perftest: Add parameter negotiation phase
Implement a negotiation phase to exchange and verify test parameters. e.g., message size, iterations, connection type, QP count... Signed-off-by: Diar Batheesh <dbatheesh@nvidia.com>
1 parent 07ece0d commit 5453511

File tree

12 files changed

+299
-2
lines changed

12 files changed

+299
-2
lines changed

‎src/atomic_bw.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,13 @@ int main(int argc, char *argv[])
143143
goto free_mem;
144144
}
145145

146+
/* Negotiate parameters. */
147+
if (negotiate_params(&ctx, &user_comm, &user_param)) {
148+
fprintf(stderr, " Failed to negotiate parameters\n");
149+
dealloc_ctx(&ctx, &user_param);
150+
goto free_mem;
151+
}
152+
146153
/* Create RDMA CM resources and connect through CM. */
147154
if (user_param.work_rdma_cm == ON) {
148155
rc = create_rdma_cm_connection(&ctx, &user_param, &user_comm,

‎src/atomic_lat.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,13 @@ int main(int argc, char *argv[])
152152
goto free_mem;
153153
}
154154

155+
/* Negotiate parameters. */
156+
if (negotiate_params(&ctx, &user_comm, &user_param)) {
157+
fprintf(stderr, " Failed to negotiate parameters\n");
158+
dealloc_ctx(&ctx, &user_param);
159+
goto free_mem;
160+
}
161+
155162
/* Create RDMA CM resources and connect through CM. */
156163
if (user_param.work_rdma_cm == ON) {
157164
rc = create_rdma_cm_connection(&ctx, &user_param, &user_comm,

‎src/perftest_communication.c‎

Lines changed: 101 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ static int post_one_recv_wqe(struct pingpong_context *ctx)
161161
struct ibv_sge list;
162162

163163
list.addr = (uintptr_t)ctx->buf[0];
164-
list.length = sizeof(struct pingpong_dest);
164+
list.length = MAX(sizeof(struct pingpong_dest), sizeof(struct perftest_parameters_negotiate));
165165
list.lkey = ctx->mr[0]->lkey;
166166

167167
wr.next = NULL;
@@ -952,6 +952,106 @@ int set_up_connection(struct pingpong_context *ctx,
952952
return 0;
953953
}
954954

955+
956+
/******************************************************************************
957+
*
958+
******************************************************************************/
959+
960+
int negotiate_params(struct pingpong_context *ctx,
961+
struct perftest_comm *comm,
962+
struct perftest_parameters *user_param)
963+
{
964+
/* Fill local parameters */
965+
struct perftest_parameters_negotiate local_params = {
966+
.test_method = hton_int(user_param->test_method),
967+
.connection_type = hton_int(user_param->connection_type),
968+
.verb = hton_int(user_param->verb),
969+
.tst = hton_int(user_param->tst),
970+
.atomicType = hton_int(user_param->atomicType),
971+
.test_type = hton_int(user_param->test_type),
972+
.report_fmt = hton_int(user_param->report_fmt),
973+
.size = hton_64(user_param->size),
974+
.iters = hton_64(user_param->iters),
975+
.aes_xts = hton_int((int)user_param->aes_xts),
976+
.num_of_qps = hton_int(user_param->num_of_qps),
977+
.duration = hton_int(user_param->duration),
978+
.use_rdma_cm = hton_int(user_param->use_rdma_cm),
979+
.use_write_with_imm = hton_int(user_param->use_write_with_imm),
980+
.no_enhanced_reorder = hton_int(user_param->no_enhanced_reorder),
981+
.sig_offload = hton_int(user_param->sig_offload),
982+
};
983+
984+
if (ibv_query_device(ctx->context, &local_params.attr)) {
985+
fprintf(stderr, " Failed to query device attributes\n");
986+
return FAILURE;
987+
}
988+
989+
#ifdef HAVE_MLX5DV
990+
if (local_params.attr.vendor_id == MLNX_VENDOR_ID) {
991+
struct mlx5dv_context ctx_dv;
992+
#ifdef HAVE_OOO_RECV_WRS
993+
ctx_dv.comp_mask = MLX5DV_CONTEXT_MASK_OOO_RECV_WRS;
994+
#endif
995+
996+
if (mlx5dv_query_device(ctx->context, &ctx_dv)){
997+
fprintf(stderr, " Failed to query device capabilities\n");
998+
return FAILURE;
999+
}
1000+
local_params.mlx5dv_comp_mask = hton_64(ctx_dv.comp_mask);
1001+
}
1002+
#endif
1003+
1004+
struct perftest_parameters_negotiate remote_params;
1005+
/* Exchange parameters */
1006+
if (ctx_xchg_data(comm, &local_params, &remote_params, sizeof(local_params))) {
1007+
fprintf(stderr, " Failed to exchange negotiation parameters between server and client\n");
1008+
return FAILURE;
1009+
}
1010+
1011+
#define COMPARE(func, name, values, type) \
1012+
{ func, #name, &local_params.name, &remote_params.name, values, type }
1013+
1014+
CompareFunction compare_functions[] = {
1015+
// COMPARE(compare_func, name, values, type) -> { compare_func, name, local_value, remote_value, return_values, type }
1016+
COMPARE(compare, test_method, ((char*[]){"RUN_REGULAR", "RUN_ALL", "RUN_INFINITELY"}), INT),
1017+
COMPARE(compare, connection_type, ((char*[]){"RC", "UC", "UD", "RawEth", "XRC", "DC", "SRD"}), INT),
1018+
COMPARE(compare, verb, ((char*[]){"SEND", "SEND_IMM", "WRITE", "WRITE_IMM", "READ", "ATOMIC"}), INT),
1019+
COMPARE(compare, tst, ((char*[]){"LAT", "BW", "LAT_BY_BW", "FS_RATE"}), INT),
1020+
COMPARE(compare, atomicType, ((char*[]){"CMP_AND_SWAP", "FETCH_AND_ADD"}), INT),
1021+
COMPARE(compare, test_type, ((char*[]){"ITERATIONS", "DURATION"}), INT),
1022+
COMPARE(compare, report_fmt, ((char*[]){"GBS", "MBS"}), INT),
1023+
COMPARE(compare, size, NULL, UINT64),
1024+
COMPARE(compare, iters, NULL, UINT64),
1025+
COMPARE(compare, aes_xts, ((char*[]){"OFF", "ON"}), INT),
1026+
COMPARE(compare, num_of_qps, NULL, INT),
1027+
COMPARE(compare, duration, NULL, INT),
1028+
COMPARE(compare, use_rdma_cm, ((char*[]){"OFF", "ON"}), INT),
1029+
COMPARE(compare, use_write_with_imm, ((char*[]){"OFF", "ON"}), INT),
1030+
COMPARE(compare, no_enhanced_reorder, ((char*[]){"OFF", "ON"}), INT),
1031+
COMPARE(compare, sig_offload, ((char*[]){"OFF", "ON"}), INT),
1032+
COMPARE(compare_ibv_device, attr, NULL, NONE),
1033+
};
1034+
#undef COMPARE
1035+
1036+
for (int i = 0; i < sizeof(compare_functions)/sizeof(compare_functions[0]); i++) {
1037+
if (compare_functions[i].compare_func(compare_functions[i].name,
1038+
compare_functions[i].local_value,
1039+
compare_functions[i].remote_value,
1040+
compare_functions[i].return_values,
1041+
user_param, compare_functions[i].type) == FAILURE){
1042+
return FAILURE;
1043+
}
1044+
}
1045+
1046+
#ifdef HAVE_MLX5DV
1047+
if (local_params.attr.vendor_id == MLNX_VENDOR_ID) {
1048+
compare_mlx5dv("mlx5dv_comp_mask", &local_params.mlx5dv_comp_mask, &remote_params.mlx5dv_comp_mask, NULL, user_param, NONE);
1049+
}
1050+
#endif
1051+
1052+
return SUCCESS;
1053+
}
1054+
9551055
/******************************************************************************
9561056
*
9571057
******************************************************************************/

‎src/perftest_communication.h‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,10 @@
4444
#include <rdma/rdma_cma.h>
4545
#include "perftest_resources.h"
4646

47+
#if defined(HAVE_MLX5DV)
48+
#include <infiniband/mlx5dv.h>
49+
#endif
50+
4751
/* Macro for 64 bit variables to switch to/from net */
4852
#if BYTE_ORDER == BIG_ENDIAN
4953
#define ntoh_64(x) (x)
@@ -161,6 +165,24 @@ int set_up_connection(struct pingpong_context *ctx,
161165
struct perftest_parameters *user_param,
162166
struct pingpong_dest *my_dest);
163167

168+
169+
/* negotiate_params .
170+
*
171+
* Description : Negotiates parameters and capabilities between server and client.
172+
*
173+
* Parameters :
174+
* ctx - Pingpong context.
175+
* comm - Communication struct.
176+
* user_param - Perftest parameters.
177+
* rem_params - Remote parameters and capabilities.
178+
*
179+
* Return Value : SUCCESS,FAILURE.
180+
*/
181+
int negotiate_params(struct pingpong_context *ctx,
182+
struct perftest_comm *comm,
183+
struct perftest_parameters *user_param);
184+
185+
164186
/* establish_connection .
165187
*
166188
* Description :
@@ -858,7 +880,6 @@ int create_rdma_cm_connection(struct pingpong_context *ctx,
858880
struct perftest_parameters *user_param, struct perftest_comm *comm,
859881
struct pingpong_dest *my_dest, struct pingpong_dest *rem_dest);
860882

861-
862883
#endif /* PERFTEST_COMMUNICATION_H */
863884

864885

‎src/perftest_parameters.c‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4845,6 +4845,72 @@ void print_report_fs_rate (struct perftest_parameters *user_param)
48454845

48464846
free(delta);
48474847
}
4848+
4849+
int compare(const char *name, void* local_value, void* remote_value, char** return_values,
4850+
struct perftest_parameters* user_param, CompareType type)
4851+
{
4852+
uint64_t local_value_int, remote_value_int;
4853+
const char *type_str = (type >= 0 && type <= 2) ? ((char*[]){"INT", "UINT64", "NONE"})[type] : "UNKNOWN";
4854+
4855+
switch (type) {
4856+
case UINT64:
4857+
local_value_int = ntoh_64(*(uint64_t*)local_value);
4858+
remote_value_int = ntoh_64(*(uint64_t*)remote_value);
4859+
break;
4860+
case INT:
4861+
local_value_int = ntoh_int(*(int*)local_value);
4862+
remote_value_int = ntoh_int(*(int*)remote_value);
4863+
break;
4864+
default:
4865+
fprintf(stderr, " Unexpected compare type %s for %s\n", type_str, name);
4866+
return FAILURE;
4867+
}
4868+
4869+
if (local_value_int == remote_value_int)
4870+
return SUCCESS;
4871+
4872+
if (return_values) {
4873+
fprintf(stderr, " %s mismatch: local=%s, remote=%s\n", name,
4874+
return_values[local_value_int], return_values[remote_value_int]);
4875+
} else {
4876+
fprintf(stderr, " %s mismatch: local=%lu, remote=%lu\n", name, local_value_int, remote_value_int);
4877+
}
4878+
4879+
return FAILURE;
4880+
}
4881+
4882+
int compare_ibv_device(const char *name, void* local_value, void* remote_value, char** return_values,
4883+
struct perftest_parameters* user_param, CompareType type)
4884+
{
4885+
struct ibv_device_attr remote_attr = *(struct ibv_device_attr*)remote_value;
4886+
4887+
if (user_param->out_reads > remote_attr.max_qp_rd_atom) { // 0 < user_param->out_reads <= local_attr.max_qp_rd_atom
4888+
printf(" out_reads is greater than remote max_qp_rd_atom, using remote max_qp_rd_atom\n");
4889+
user_param->out_reads = remote_attr.max_qp_rd_atom;
4890+
}
4891+
4892+
return SUCCESS;
4893+
}
4894+
4895+
#ifdef HAVE_MLX5DV
4896+
int compare_mlx5dv(const char *name, void* local_value, void* remote_value, char** return_values,
4897+
struct perftest_parameters* user_param, CompareType type)
4898+
{
4899+
#ifdef HAVE_OOO_RECV_WRS
4900+
uint64_t local_value_uint64 = ntoh_64(*(uint64_t*)local_value);
4901+
uint64_t remote_value_uint64 = ntoh_64(*(uint64_t*)remote_value);
4902+
4903+
if ( user_param->no_enhanced_reorder != ON &&
4904+
(local_value_uint64 & MLX5DV_CONTEXT_MASK_OOO_RECV_WRS) != (remote_value_uint64 & MLX5DV_CONTEXT_MASK_OOO_RECV_WRS)) {
4905+
user_param->no_enhanced_reorder = ON;
4906+
printf(" OOO_RECV_WRS mismatch, disabling enhanced reorder\n");
4907+
}
4908+
4909+
#endif
4910+
return SUCCESS;
4911+
}
4912+
#endif
4913+
48484914
/******************************************************************************
48494915
* End
48504916
******************************************************************************/

‎src/perftest_parameters.h‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,6 +323,8 @@
323323
} \
324324
} while (0)
325325

326+
typedef enum { INT, UINT64, NONE } CompareType;
327+
326328
/* The Verb of the benchmark. */
327329
typedef enum { SEND, SEND_IMM, WRITE, WRITE_IMM, READ, ATOMIC } VerbType;
328330

@@ -708,6 +710,39 @@ struct bw_report_data {
708710
int sl;
709711
};
710712

713+
struct perftest_parameters_negotiate {
714+
enum ctx_test_method test_method;
715+
int connection_type;
716+
VerbType verb;
717+
TestType tst;
718+
AtomicType atomicType;
719+
TestMethod test_type;
720+
enum ctx_report_fmt report_fmt;
721+
uint64_t size;
722+
uint64_t iters;
723+
int aes_xts;
724+
int num_of_qps;
725+
int duration;
726+
int use_rdma_cm;
727+
int use_write_with_imm;
728+
int no_enhanced_reorder;
729+
int sig_offload;
730+
struct ibv_device_attr attr;
731+
732+
#ifdef HAVE_MLX5DV
733+
uint64_t mlx5dv_comp_mask;
734+
#endif
735+
};
736+
737+
typedef struct {
738+
int (*compare_func)(const char *, void*, void*, char**, struct perftest_parameters*, CompareType type);
739+
const char *name;
740+
void* local_value;
741+
void* remote_value;
742+
char** return_values;
743+
CompareType type;
744+
} CompareFunction;
745+
711746
struct rate_gbps_string {
712747
enum ibv_rate rate_gbps_enum;
713748
char* rate_gbps_str;
@@ -910,4 +945,23 @@ int set_eth_mtu(struct perftest_parameters *user_param);
910945
******************************************************************************/
911946
enum ctx_device ib_dev_name(struct ibv_context *context);
912947

948+
/********************************************************************************
949+
*
950+
*********************************************************************************/
951+
int compare(const char *name, void* local_value, void* remote_value, char** return_values,
952+
struct perftest_parameters* user_param, CompareType type);
953+
954+
/********************************************************************************
955+
*
956+
*********************************************************************************/
957+
int compare_ibv_device(const char *name, void* local_value, void* remote_value, char** return_values,
958+
struct perftest_parameters* user_param, CompareType type);
959+
960+
/********************************************************************************
961+
*
962+
*********************************************************************************/
963+
#ifdef HAVE_MLX5DV
964+
int compare_mlx5dv(const char *name, void* local_value, void* remote_value, char** return_values,
965+
struct perftest_parameters* user_param, CompareType type);
966+
#endif
913967
#endif /* PERFTEST_RESOURCES_H */

‎src/read_bw.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,13 @@ int main(int argc, char *argv[])
141141
goto free_mem;
142142
}
143143

144+
/* Negotiate parameters. */
145+
if (negotiate_params(&ctx, &user_comm, &user_param)) {
146+
fprintf(stderr, " Failed to negotiate parameters\n");
147+
dealloc_ctx(&ctx, &user_param);
148+
goto free_mem;
149+
}
150+
144151
/* Create RDMA CM resources and connect through CM. */
145152
if (user_param.work_rdma_cm == ON) {
146153
rc = create_rdma_cm_connection(&ctx, &user_param, &user_comm,

‎src/read_lat.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,13 @@ int main(int argc, char *argv[])
152152
goto free_mem;
153153
}
154154

155+
/* Negotiate parameters. */
156+
if (negotiate_params(&ctx, &user_comm, &user_param)) {
157+
fprintf(stderr, " Failed to negotiate parameters\n");
158+
dealloc_ctx(&ctx, &user_param);
159+
goto free_mem;
160+
}
161+
155162
/* Create RDMA CM resources and connect through CM. */
156163
if (user_param.work_rdma_cm == ON) {
157164
rc = create_rdma_cm_connection(&ctx, &user_param, &user_comm,

‎src/send_bw.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -268,6 +268,13 @@ int main(int argc, char *argv[])
268268
goto free_mem;
269269
}
270270

271+
/* Negotiate parameters. */
272+
if (negotiate_params(&ctx, &user_comm, &user_param)) {
273+
fprintf(stderr, " Failed to negotiate parameters\n");
274+
dealloc_ctx(&ctx, &user_param);
275+
goto free_mem;
276+
}
277+
271278
/* Create RDMA CM resources and connect through CM. */
272279
if (user_param.work_rdma_cm == ON) {
273280
rc = create_rdma_cm_connection(&ctx, &user_param, &user_comm,

‎src/send_lat.c‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -272,6 +272,13 @@ int main(int argc, char *argv[])
272272
goto free_mem;
273273
}
274274

275+
/* Negotiate parameters. */
276+
if (negotiate_params(&ctx, &user_comm, &user_param)) {
277+
fprintf(stderr, " Failed to negotiate parameters\n");
278+
dealloc_ctx(&ctx, &user_param);
279+
goto free_mem;
280+
}
281+
275282
/* Create RDMA CM resources and connect through CM. */
276283
if (user_param.work_rdma_cm == ON) {
277284
rc = create_rdma_cm_connection(&ctx, &user_param, &user_comm,

0 commit comments

Comments
 (0)