Skip to content

Commit cd2ff53

Browse files
kevintraynorkevintraynor
committed
dpdk: Convert dpdk-lcore-mask to DPDK lcore args.
OVS currently uses other_config:dpdk-lcore-mask <coremask> directly in DPDK rte_eal_init() with '-c <coremask>' argument. '-c' argument is now deprecated from DPDK and will be removed in DPDK 25.11, so OVS will no longer be able to use the '-c <coremask>' argument. Convert dpdk-lcore-mask core mask to a core list that can be used with '--lcores' and add some tests. The core list is validated to prevent invalid cores being passed to DPDK rte_eal_init(). Using the '--lcores' argument also adds compatibility for using a core in the core mask that is greater than the max lcore, similar to commit fe53b47 ("dpdk: Fix main lcore on systems with many cores.") Signed-off-by: Kevin Traynor <ktraynor@redhat.com> Acked-by: Eelco Chaudron <echaudro@redhat.com> Reviewed-by: David Marchand <david.marchand@redhat.com>
1 parent 7bf5a8b commit cd2ff53

File tree

4 files changed

+160
-19
lines changed

4 files changed

+160
-19
lines changed

‎lib/dpdk.c‎

Lines changed: 72 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,16 +64,71 @@ args_contains(const struct svec *args, const char *value)
6464
return false;
6565
}
6666

67+
static int
68+
compare_core_ids(const void *a_, const void *b_)
69+
{
70+
const unsigned int *a = a_;
71+
const unsigned int *b = b_;
72+
73+
return *a < *b ? -1 : *a > *b;
74+
}
75+
76+
/* Converts a hexadecimal core mask to DPDK lcore list format.
77+
* Returns NULL if no valid bits are set in the mask. */
78+
static char *
79+
cmask_to_lcore_list(const char *cmask)
80+
{
81+
struct ds ds = DS_EMPTY_INITIALIZER;
82+
struct ovs_numa_dump *lcore_dump;
83+
struct ovs_numa_info_core *core;
84+
unsigned int *core_ids = NULL;
85+
unsigned int num_cores;
86+
unsigned int i = 0;
87+
88+
lcore_dump = ovs_numa_dump_cores_with_cmask(cmask);
89+
90+
num_cores = ovs_numa_dump_count(lcore_dump);
91+
if (num_cores == 0) {
92+
ovs_numa_dump_destroy(lcore_dump);
93+
return NULL;
94+
}
95+
core_ids = xmalloc(num_cores * sizeof *core_ids);
96+
97+
FOR_EACH_CORE_ON_DUMP (core, lcore_dump) {
98+
core_ids[i++] = core->core_id;
99+
}
100+
101+
/* ovs_numa_dump uses a hash map, so sort by core id. */
102+
qsort(core_ids, num_cores, sizeof *core_ids, compare_core_ids);
103+
104+
if (num_cores > RTE_MAX_LCORE) {
105+
VLOG_INFO("Truncating DPDK lcore list as dpdk-lcore-mask has >"
106+
" %u lcores", RTE_MAX_LCORE);
107+
num_cores = RTE_MAX_LCORE;
108+
}
109+
110+
for (i = 0; i < num_cores; i++) {
111+
ds_put_format(&ds, "%u@%u,", i, core_ids[i]);
112+
}
113+
/* Remove trailing comma. */
114+
ds_chomp(&ds, ',');
115+
116+
ovs_numa_dump_destroy(lcore_dump);
117+
free(core_ids);
118+
return ds_steal_cstr(&ds);
119+
}
120+
67121
static void
68122
construct_dpdk_options(const struct smap *ovs_other_config, struct svec *args)
69123
{
70124
struct dpdk_options_map {
71125
const char *ovs_configuration;
72126
const char *dpdk_option;
127+
char *(*param_conversion) (const char *);
73128
} opts[] = {
74-
{"dpdk-lcore-mask", "-c" },
75-
{"dpdk-hugepage-dir", "--huge-dir" },
76-
{"dpdk-socket-limit", "--socket-limit"},
129+
{"dpdk-lcore-mask", "--lcores", cmask_to_lcore_list},
130+
{"dpdk-hugepage-dir", "--huge-dir", NULL},
131+
{"dpdk-socket-limit", "--socket-limit", NULL},
77132
};
78133

79134
int i;
@@ -85,11 +140,24 @@ construct_dpdk_options(const struct smap *ovs_other_config, struct svec *args)
85140

86141
if (value) {
87142
if (!args_contains(args, opts[i].dpdk_option)) {
143+
char *dpdk_val = NULL;
144+
145+
if (opts[i].param_conversion) {
146+
dpdk_val = (opts[i].param_conversion)(value);
147+
if (!dpdk_val) {
148+
VLOG_WARN("Ignoring database defined option '%s'"
149+
" due to invalid value '%s'",
150+
opts[i].ovs_configuration, value);
151+
continue;
152+
}
153+
value = dpdk_val;
154+
}
88155
svec_add(args, opts[i].dpdk_option);
89156
svec_add(args, value);
157+
free(dpdk_val);
90158
} else {
91159
VLOG_WARN("Ignoring database defined option '%s' due to "
92-
"dpdk-extra config", opts[i].dpdk_option);
160+
"dpdk-extra config", opts[i].ovs_configuration);
93161
}
94162
}
95163
}

‎tests/ovs-macros.at‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -416,3 +416,19 @@ m4_define([XT6_ACCEPT],
416416
else
417417
IPTABLES_ACCEPT([ip6tables], [$1])
418418
fi])
419+
420+
dnl CHECK_CPU_DISCOVERED([n_cpu])
421+
dnl
422+
dnl Waits until CPUs are discovered and checks if the number of discovered CPUs
423+
dnl is greater or equal to 'n_cpu'. Without the 'n_cpu' parameter checks that
424+
dnl at least one CPU is discovered.
425+
m4_define([CHECK_CPU_DISCOVERED], [
426+
PATTERN="Discovered [[0-9]]* NUMA nodes and [[0-9]]* CPU cores"
427+
OVS_WAIT_UNTIL([grep "$PATTERN" ovs-vswitchd.log])
428+
N_CPU=$(grep "$PATTERN" ovs-vswitchd.log \
429+
| sed -e 's/.* \([[0-9]]*\) CPU cores/\1/')
430+
if [[ -z "$1" ]]
431+
then AT_CHECK([test "$N_CPU" -gt "0"])
432+
else AT_SKIP_IF([test "$N_CPU" -lt "$1"])
433+
fi
434+
])

‎tests/pmd.at‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -26,21 +26,6 @@ flow_dump_prepend_pmd () {
2626

2727
m4_divert_pop([PREPARE_TESTS])
2828

29-
dnl CHECK_CPU_DISCOVERED([n_cpu])
30-
dnl
31-
dnl Waits until CPUs discovered and checks if number of discovered CPUs
32-
dnl is greater or equal to 'n_cpu'. Without parameters checks that at
33-
dnl least one CPU discovered.
34-
m4_define([CHECK_CPU_DISCOVERED], [
35-
PATTERN="Discovered [[0-9]]* NUMA nodes and [[0-9]]* CPU cores"
36-
OVS_WAIT_UNTIL([grep "$PATTERN" ovs-vswitchd.log])
37-
N_CPU=$(grep "$PATTERN" ovs-vswitchd.log | sed -e 's/.* \([[0-9]]*\) CPU cores/\1/')
38-
if [[ -z "$1" ]]
39-
then AT_CHECK([test "$N_CPU" -gt "0"])
40-
else AT_SKIP_IF([test "$N_CPU" -lt "$1"])
41-
fi
42-
])
43-
4429
dnl CHECK_PMD_THREADS_CREATED([n_threads], [numa_id], [+line])
4530
dnl
4631
dnl Whaits for creation of 'n_threads' or at least 1 thread if $1 not

‎tests/system-dpdk.at‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,78 @@ dnl --------------------------------------------------------------------------
5353

5454

5555

56+
dnl --------------------------------------------------------------------------
57+
dnl Check dpdk-lcore-mask conversion for only first bit
58+
AT_SETUP([OVS-DPDK - dpdk-lcore-mask conversion - single])
59+
AT_KEYWORDS([dpdk])
60+
OVS_DPDK_PRE_CHECK()
61+
OVS_DPDK_START_OVSDB()
62+
OVS_DPDK_START_VSWITCHD([--no-pci])
63+
CHECK_CPU_DISCOVERED()
64+
AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0x1])
65+
AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true])
66+
expected_lcores="lcores 0@0"
67+
OVS_WAIT_UNTIL([grep "$expected_lcores" ovs-vswitchd.log])
68+
OVS_DPDK_STOP_VSWITCHD
69+
AT_CLEANUP
70+
dnl --------------------------------------------------------------------------
71+
72+
73+
74+
dnl --------------------------------------------------------------------------
75+
dnl Check dpdk-lcore-mask conversion for only multiple bits
76+
AT_SETUP([OVS-DPDK - dpdk-lcore-mask conversion - multi])
77+
AT_KEYWORDS([dpdk])
78+
OVS_DPDK_PRE_CHECK()
79+
OVS_DPDK_START_OVSDB()
80+
OVS_DPDK_START_VSWITCHD([--no-pci])
81+
CHECK_CPU_DISCOVERED(4)
82+
AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0xf])
83+
AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true])
84+
expected_lcores="lcores 0@0,1@1,2@2,3@3"
85+
OVS_WAIT_UNTIL([grep "$expected_lcores" ovs-vswitchd.log])
86+
OVS_DPDK_STOP_VSWITCHD
87+
AT_CLEANUP
88+
dnl --------------------------------------------------------------------------
89+
90+
91+
92+
dnl --------------------------------------------------------------------------
93+
dnl Check dpdk-lcore-mask conversion for max length string
94+
AT_SETUP([OVS-DPDK - dpdk-lcore-mask conversion - non-contig])
95+
AT_KEYWORDS([dpdk])
96+
OVS_DPDK_PRE_CHECK()
97+
OVS_DPDK_START_OVSDB()
98+
OVS_DPDK_START_VSWITCHD([--no-pci])
99+
CHECK_CPU_DISCOVERED(8)
100+
AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0xca])
101+
AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true])
102+
expected_lcores="lcores 0@1,1@3,2@6,3@7"
103+
OVS_WAIT_UNTIL([grep "$expected_lcores" ovs-vswitchd.log])
104+
OVS_DPDK_STOP_VSWITCHD
105+
AT_CLEANUP
106+
dnl --------------------------------------------------------------------------
107+
108+
109+
110+
dnl --------------------------------------------------------------------------
111+
dnl Check dpdk-lcore-mask conversion for zero
112+
AT_SETUP([OVS-DPDK - dpdk-lcore-mask conversion - zeromask])
113+
AT_KEYWORDS([dpdk])
114+
OVS_DPDK_PRE_CHECK()
115+
OVS_DPDK_START_OVSDB()
116+
OVS_DPDK_START_VSWITCHD([--no-pci])
117+
AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask=0x0])
118+
AT_CHECK([ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true])
119+
OVS_WAIT_UNTIL([grep "Ignoring database defined option 'dpdk-lcore-mask' due to invalid value '0x0'" ovs-vswitchd.log])
120+
expected_lcores="lcores 0@0"
121+
OVS_WAIT_UNTIL([grep "$expected_lcores" ovs-vswitchd.log])
122+
OVS_DPDK_STOP_VSWITCHD(["/Ignoring database defined option/d"])
123+
AT_CLEANUP
124+
dnl --------------------------------------------------------------------------
125+
126+
127+
56128
dnl --------------------------------------------------------------------------
57129
dnl Add standard DPDK PHY port
58130
AT_SETUP([OVS-DPDK - add standard DPDK port])

0 commit comments

Comments
 (0)