Memory sharing between DPDK Multi-process
Refer: https://doc.dpdk.org/guides/prog_guide/multi_proc_support.html
DPDK-Procinfo as secondary process
Default DPDK EAL arguments for dpdk-procinfo are -c1, -n4 & --proc-type=secondary
Refer: https://doc.dpdk.org/guides-18.08/tools/proc_info.html
Modify dpdk-helloworld to create mempool and rte-ring
DPDK helloword application “dpdk-stable-19.11.2/examples/helloworld” is a primary proc type. This is modified to create a rte-mempool, and a rte_ring to use that mempool to enqueue and dequeue message from mempool. This mempool is shared with secondary process (in this case dpdk-procinfo)
/* SPDX-License-Identifier: BSD-3-Clause
* Copyright(c) 2010-2014 Intel Corporation
*/
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <errno.h>
#include <sys/queue.h>
#include <rte_memory.h>
#include <rte_launch.h>
#include <rte_eal.h>
#include <rte_per_lcore.h>
#include <rte_lcore.h>
#include <rte_debug.h>
#include <rte_ring.h>
#include <rte_mempool.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/time.h>
#define STR_TOKEN_SIZE 128
static const char *_MSG_POOL = "MSG_POOL";
static const char *_SEC_2_PRI = "SEC_2_PRI";
static const char *_PRI_2_SEC = "PRI_2_SEC";
struct rte_ring *send_ring, *recv_ring;
struct rte_mempool *message_pool;
struct counter_s{
long int tv_sec;
unsigned countA;
unsigned countB;
unsigned countC;
unsigned countD;
};
struct counter_s count;
time_t curtime;
static int
lcore_hello(__attribute__((unused)) void *arg)
{
unsigned lcore_id;
lcore_id = rte_lcore_id();
printf("hello from core %u socker %u\n", lcore_id, rte_socket_id());
return 0;
}
int
main(int argc, char **argv)
{
int ret;
unsigned lcore_id;
const unsigned flags = 10;
const unsigned ring_size = 64;
//const unsigned pool_size = 1024;
const unsigned pool_size = 1024;
const unsigned pool_cache = 32;
const unsigned priv_data_sz = 0;
ret = rte_eal_init(argc, argv);
if (ret < 0)
rte_panic("Cannot init EAL\n");
if (rte_eal_process_type() == RTE_PROC_PRIMARY){
send_ring = rte_ring_create(_PRI_2_SEC, ring_size, rte_socket_id(), flags);
recv_ring = rte_ring_create(_SEC_2_PRI, ring_size, rte_socket_id(), flags);
message_pool = rte_mempool_create(_MSG_POOL,
/* total number of entry =*/pool_size,
/* size of each entry =*/256,
pool_cache,
priv_data_sz,
NULL, NULL, NULL, NULL,
rte_socket_id(), flags);
} else {
message_pool = rte_mempool_lookup(_MSG_POOL);
}
if (send_ring == NULL)
rte_exit(EXIT_FAILURE, "Problem getting sending ring\n");
if (recv_ring == NULL)
rte_exit(EXIT_FAILURE, "Problem getting receiving ring\n");
if (message_pool == NULL)
rte_exit(EXIT_FAILURE, "Problem getting message pool\n");
/* call lcore_hello() on every slave lcore */
RTE_LCORE_FOREACH_SLAVE(lcore_id) {
rte_eal_remote_launch(lcore_hello, NULL, lcore_id);
}
/* call it on master lcore too */
lcore_hello(NULL);
while(1)
{
struct timeval tv;
gettimeofday(&tv, NULL);
count.tv_sec = tv.tv_sec;
count.countA = rand() % 2200;
count.countB = rand() % 300;
count.countC = rand() % 78;
count.countD = rand() % 1300;
void *msg = NULL;
if (rte_mempool_get(message_pool, &msg) < 0)
rte_panic("Failed to get message buffer\n");
memcpy(msg, &count, sizeof(count));
if (rte_ring_enqueue(send_ring, msg) < 0)
{
printf("Engueuq failed\n");
rte_mempool_put(message_pool, msg);
}
sleep(10);
}
rte_eal_mp_wait_lcore();
return 0;
}
Modification in DPDK-procinfo
DPDK-procinfo “dpdk-stable-19.11.2/app/proc-info” is modified to to get message from memool published from primary procee (in this case dpdk-helloworld)
static void
show_ring(char *tname)
{
snprintf(bdr_str, MAX_STRING_LEN, " show - RING %"PRIu64,
rte_get_tsc_hz());
STATS_BDR_STR(10, bdr_str);
char *name = strtok(tname, ",");
char *mempool = strtok(NULL, ",");
FILE *fp;
//printf("ring:%s, mempool:%s\n", name, mempool);
if (name != NULL) {
struct rte_ring *ptr = rte_ring_lookup(name);
if (ptr != NULL) {
printf(" - Name (%s) on socket (%d)\n"
" - flags:\n"
"\t -- Single Producer Enqueue (%u)\n"
"\t -- Single Consmer Dequeue (%u)\n",
ptr->name,
ptr->memzone->socket_id,
ptr->flags & RING_F_SP_ENQ,
ptr->flags & RING_F_SC_DEQ);
printf(" - size (%u) mask (0x%x) capacity (%u)\n",
ptr->size,
ptr->mask,
ptr->capacity);
printf(" - count (%u) free count (%u)\n",
rte_ring_count(ptr),
rte_ring_free_count(ptr));
printf(" - full (%d) empty (%d)\n",
rte_ring_full(ptr),
rte_ring_empty(ptr));
STATS_BDR_STR(50, "");
//RTE_LCORE_FOREACH_SLAVE() {
if(rte_ring_count(ptr))
{
fp = fopen("./test.txt", "w+");
while(rte_ring_count(ptr))
{
void *msg;
if (rte_ring_dequeue(ptr, &msg) < 0)
{
printf("No msh recv");
}
else
{
struct counter_s* count = (struct counter_s*)msg;
if(count)
fprintf(fp,"%ld,%u,%u,%u,%u\n",count->tv_sec,
count->countA,
count->countB,
count->countC,
count->countD);
if (mempool != NULL) {
struct rte_mempool *mptr = rte_mempool_lookup(mempool);
printf(" Got mempool\n");
if (mptr != NULL) {
printf("calling rte_mempool_put\n");
rte_mempool_put(mptr, msg);
}
}
}
}
fclose(fp);
}
return;
}
}
rte_ring_list_dump(stdout);
STATS_BDR_STR(50, "");
}
Compilation and run
//get
wget http://fast.dpdk.org/rel/dpdk-19.11.2.tar.xz
tar xvf dpdk-19.11.2.tar.xz
//basic library for dpdk
sudo apt install make
sudo apt install gcc
sudo apt-get install libnuma-dev (numa.h)
sudo apt-get install libpcap0.8-dev (pcap.h, useful sometimes)
sudo apt install dpdk
//dpdk-devbind.py -s (test the devbind)
//set-up and build dpdk library
sudo bash
cd <>/dpdk-stable-19.11.2/usertools
export DESTDIR=<>/dpdk-stable-19.11.2
#build (from <>/dpdk-<>/usertools)
./dpdk-setup.sh
1. x86_64-native-linuxapp-gcc
2. Setup hugepage mappings for NUMA systems
2 MB pages
Number of pages for node0: 50
Number of pages for node1: 50
//compile helloworld example
export RTE_SDK=<>/dpdk-stable-19.11.2
export RTE_TARGET=x86_64-native-linuxapp-gcc
cd examples/helloworld/
make
sudo ./helloworld -l 0-3 -n 4
//compile dpdk-proc info
export RTE_SDK=<>/dpdk-stable-19.11.2
export RTE_TARGET=x86_64-native-linuxapp-gcc
cd <>/dpdk-stable-19.11.2/app/proc-info
make
sudo ./dpdk-procinfo -- --show-ring=PRI_2_SECOutput
//helloworld
sudo ./build/helloworld -l 0-3 -n 4
EAL: Detected 6 lcore(s)
EAL: Detected 2 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: No available hugepages reported in hugepages-1048576kB
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: PCI device 0000:19:00.0 on NUMA socket 0
...
EAL: PCI device 0000:88:00.3 on NUMA socket 1
EAL: probe driver: 8086:1572 net_i40e
hello from core 1 socker 1
hello from core 2 socker 0
hello from core 3 socker 1
hello from core 0 socker 0
//dpdk-procinfo ring
sudo ./dpdk-procinfo -- --show-ring=PRI_2_SEC
EAL: Detected 6 lcore(s)
EAL: Detected 2 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket_199753_59cb4dbfe0fb4
EAL: Selected IOVA mode 'PA'
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: PCI device 0000:19:00.0 on NUMA socket 0
EAL: probe driver: 8086:1572 net_i40e
....
EAL: PCI device 0000:88:00.3 on NUMA socket 1
EAL: probe driver: 8086:1572 net_i40e
========== show - RING 2690000000==========
- Name (PRI_2_SEC) on socket (0)
- flags:
-- Single Producer Enqueue (0)
-- Single Consmer Dequeue (2)
- size (64) mask (0x3f) capacity (63)
- count (6) free count (57)
- full (0) empty (0)
================================================================================
cat test.txt
1594269122,1183,286,63,615
1594269132,793,235,64,692
1594269142,2049,121,2,627
1594269152,1490,259,77,326
1594269162,1340,126,10,336
1594269172,1011,68,3,29
DPDK Mempool fundamental
Refer: https://doc.dpdk.org/guides/sample_app_ug/multi_process.html
Refer: https://doc.dpdk.org/dts/test_plans/external_mempool_handler_test_plan.html
Mempool test application
dpdk-stable-19.11.2/app/test
test -n 4 -c f
RTE>> mempool_autotest
RTE>>mempool_autotest
test_mempool ret = 2111
Testing ring_mp_mc mempool handler
Walk into mempools:
test_nocache
test_cache
test_stack_anon
test_iter_obj
test_stack
default_pool
mempool <test_nocache>@0x100240800
SciTech News 