From patchwork Wed Jan 20 14:06:15 2021 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: Lijun Ou X-Patchwork-Id: 86988 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@inbox.dpdk.org Delivered-To: patchwork@inbox.dpdk.org Received: from mails.dpdk.org (mails.dpdk.org [217.70.189.124]) by inbox.dpdk.org (Postfix) with ESMTP id 935B3A0A05; Wed, 20 Jan 2021 15:06:35 +0100 (CET) Received: from [217.70.189.124] (localhost [127.0.0.1]) by mails.dpdk.org (Postfix) with ESMTP id 2738F140E02; Wed, 20 Jan 2021 15:06:35 +0100 (CET) Received: from szxga05-in.huawei.com (szxga05-in.huawei.com [45.249.212.191]) by mails.dpdk.org (Postfix) with ESMTP id 9E63E140DE2 for ; Wed, 20 Jan 2021 15:06:33 +0100 (CET) Received: from DGGEMS404-HUB.china.huawei.com (unknown [172.30.72.60]) by szxga05-in.huawei.com (SkyGuard) with ESMTP id 4DLS3906hKzj9Q6; Wed, 20 Jan 2021 22:05:25 +0800 (CST) Received: from localhost.localdomain (10.69.192.56) by DGGEMS404-HUB.china.huawei.com (10.3.19.204) with Microsoft SMTP Server id 14.3.498.0; Wed, 20 Jan 2021 22:06:30 +0800 From: Lijun Ou To: , CC: , Date: Wed, 20 Jan 2021 22:06:15 +0800 Message-ID: <1611151575-10095-1-git-send-email-oulijun@huawei.com> X-Mailer: git-send-email 2.7.4 In-Reply-To: <1610099184-37328-1-git-send-email-oulijun@huawei.com> References: <1610099184-37328-1-git-send-email-oulijun@huawei.com> MIME-Version: 1.0 X-Originating-IP: [10.69.192.56] X-CFilter-Loop: Reflected Subject: [dpdk-dev] [RFC V2] app/testpmd: support multi-process X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" This patch adds multi-process support for testpmd. The test cmd example as follows: the primary cmd: ./testpmd -w xxx --file-prefix=xx -l 0-1 -n 2 -- -i\ --rxq=16 --txq=16 --num-procs=2 --proc-id=0 the secondary cmd: ./testpmd -w xxx --file-prefix=xx -l 2-3 -n 2 -- -i\ --rxq=16 --txq=16 --num-procs=2 --proc-id=1 Signed-off-by: Min Hu (Connor) Signed-off-by: Lijun Ou --- app/test-pmd/cmdline.c | 6 ++- app/test-pmd/config.c | 9 +++- app/test-pmd/parameters.c | 9 ++++ app/test-pmd/testpmd.c | 135 +++++++++++++++++++++++++++++----------------- app/test-pmd/testpmd.h | 7 +++ 5 files changed, 114 insertions(+), 52 deletions(-) diff --git a/app/test-pmd/cmdline.c b/app/test-pmd/cmdline.c index 89034c8..48af5cd 100644 --- a/app/test-pmd/cmdline.c +++ b/app/test-pmd/cmdline.c @@ -71,8 +71,6 @@ #include "cmdline_tm.h" #include "bpf_cmd.h" -static struct cmdline *testpmd_cl; - static void cmd_reconfig_device_queue(portid_t id, uint8_t dev, uint8_t queue); /* *** Help command with introduction. *** */ @@ -17124,6 +17122,10 @@ prompt(void) if (testpmd_cl == NULL) return; cmdline_interact(testpmd_cl); + if (unlikely(f_quit == 1)) { + dup2(testpmd_fd_copy, testpmd_cl->s_in); + close(testpmd_fd_copy); + } cmdline_stdin_exit(testpmd_cl); } diff --git a/app/test-pmd/config.c b/app/test-pmd/config.c index 0e2b9f7..f065008 100644 --- a/app/test-pmd/config.c +++ b/app/test-pmd/config.c @@ -3100,6 +3100,8 @@ rss_fwd_config_setup(void) queueid_t rxq; queueid_t nb_q; streamid_t sm_id; + int start; + int end; nb_q = nb_rxq; if (nb_q > nb_txq) @@ -3117,7 +3119,10 @@ rss_fwd_config_setup(void) init_fwd_streams(); setup_fwd_config_of_each_lcore(&cur_fwd_config); - rxp = 0; rxq = 0; + start = proc_id * nb_q / num_procs; + end = start + nb_q / num_procs; + rxp = 0; + rxq = start; for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { struct fwd_stream *fs; @@ -3134,6 +3139,8 @@ rss_fwd_config_setup(void) continue; rxp = 0; rxq++; + if (rxq >= end) + rxq = start; } } diff --git a/app/test-pmd/parameters.c b/app/test-pmd/parameters.c index df5eb10..ac63854 100644 --- a/app/test-pmd/parameters.c +++ b/app/test-pmd/parameters.c @@ -45,6 +45,8 @@ #include #include "testpmd.h" +#define PARAM_PROC_ID "proc-id" +#define PARAM_NUM_PROCS "num-procs" static void usage(char* progname) @@ -603,6 +605,8 @@ launch_args_parse(int argc, char** argv) { "rx-mq-mode", 1, 0, 0 }, { "record-core-cycles", 0, 0, 0 }, { "record-burst-stats", 0, 0, 0 }, + { PARAM_NUM_PROCS, 1, 0, 0 }, + { PARAM_PROC_ID, 1, 0, 0 }, { 0, 0, 0, 0 }, }; @@ -1356,6 +1360,11 @@ launch_args_parse(int argc, char** argv) record_core_cycles = 1; if (!strcmp(lgopts[opt_idx].name, "record-burst-stats")) record_burst_stats = 1; + + if (strncmp(lgopts[opt_idx].name, PARAM_NUM_PROCS, 8) == 0) + num_procs = atoi(optarg); + if (strncmp(lgopts[opt_idx].name, PARAM_PROC_ID, 7) == 0) + proc_id = atoi(optarg); break; case 'h': usage(argv[0]); diff --git a/app/test-pmd/testpmd.c b/app/test-pmd/testpmd.c index c256e71..3abd080 100644 --- a/app/test-pmd/testpmd.c +++ b/app/test-pmd/testpmd.c @@ -63,6 +63,8 @@ #include "testpmd.h" +int testpmd_fd_copy = 500; /* the copy of STDIN_FILENO */ + #ifndef MAP_HUGETLB /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ #define HUGE_FLAG (0x40000) @@ -125,6 +127,9 @@ uint8_t port_numa[RTE_MAX_ETHPORTS]; */ uint8_t rxring_numa[RTE_MAX_ETHPORTS]; +int proc_id = 0; +unsigned num_procs = 1; + /* * Store specified sockets on which TX ring to be used by ports * is allocated. @@ -978,16 +983,26 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, /* wrapper to rte_mempool_create() */ TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", rte_mbuf_best_mempool_ops()); - rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, - mb_mempool_cache, 0, mbuf_seg_size, socket_id); + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_mp = rte_pktmbuf_pool_create(pool_name, + nb_mbuf, mb_mempool_cache, 0, + mbuf_seg_size, socket_id); + else + rte_mp = rte_mempool_lookup(pool_name); + break; } case MP_ALLOC_ANON: { - rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, - mb_size, (unsigned int) mb_mempool_cache, - sizeof(struct rte_pktmbuf_pool_private), - socket_id, mempool_flags); + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_mp = rte_mempool_create_empty(pool_name, + nb_mbuf, mb_size, + (unsigned int)mb_mempool_cache, + sizeof(struct rte_pktmbuf_pool_private), + socket_id, mempool_flags); + else + rte_mp = rte_mempool_lookup(pool_name); + if (rte_mp == NULL) goto err; @@ -1017,9 +1032,13 @@ mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", rte_mbuf_best_mempool_ops()); - rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, - mb_mempool_cache, 0, mbuf_seg_size, - heap_socket); + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_mp = rte_pktmbuf_pool_create(pool_name, + nb_mbuf, mb_mempool_cache, 0, + mbuf_seg_size, heap_socket); + else + rte_mp = rte_mempool_lookup(pool_name); + break; } case MP_ALLOC_XBUF: @@ -2503,21 +2522,28 @@ start_port(portid_t pid) return -1; } /* configure port */ - diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + diag = rte_eth_dev_configure(pi, + nb_rxq + nb_hairpinq, nb_txq + nb_hairpinq, &(port->dev_conf)); - if (diag != 0) { - if (rte_atomic16_cmpset(&(port->port_status), - RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) - printf("Port %d can not be set back " - "to stopped\n", pi); - printf("Fail to configure port %d\n", pi); - /* try to reconfigure port next time */ - port->need_reconfig = 1; - return -1; + if (diag != 0) { + if (rte_atomic16_cmpset( + &(port->port_status), + RTE_PORT_HANDLING, + RTE_PORT_STOPPED) == 0) + printf("Port %d can not be set " + "back to stopped\n", pi); + printf("Fail to configure port %d\n", + pi); + /* try to reconfigure port next time */ + port->need_reconfig = 1; + return -1; + } } } - if (port->need_reconfig_queues > 0) { + if (port->need_reconfig_queues > 0 && + rte_eal_process_type() == RTE_PROC_PRIMARY) { port->need_reconfig_queues = 0; /* setup tx queues */ for (qi = 0; qi < nb_txq; qi++) { @@ -2618,15 +2644,18 @@ start_port(portid_t pid) cnt_pi++; /* start port */ - if (rte_eth_dev_start(pi) < 0) { - printf("Fail to start port %d\n", pi); - - /* Fail to setup rx queue, return */ - if (rte_atomic16_cmpset(&(port->port_status), - RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) - printf("Port %d can not be set back to " - "stopped\n", pi); - continue; + if (rte_eal_process_type() == RTE_PROC_PRIMARY) { + diag = rte_eth_dev_start(pi); + if (diag < 0) { + printf("Fail to start port %d\n", pi); + + /* Fail to setup rx queue, return */ + if (rte_atomic16_cmpset(&(port->port_status), + RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) + printf("Port %d can not be set back to " + "stopped\n", pi); + continue; + } } if (rte_atomic16_cmpset(&(port->port_status), @@ -2755,7 +2784,7 @@ stop_port(portid_t pid) if (port->flow_list) port_flow_flush(pi); - if (rte_eth_dev_stop(pi) != 0) + if (rte_eal_process_type() == RTE_PROC_PRIMARY && rte_eth_dev_stop(pi) != 0) RTE_LOG(ERR, EAL, "rte_eth_dev_stop failed for port %u\n", pi); @@ -2824,8 +2853,10 @@ close_port(portid_t pid) continue; } - port_flow_flush(pi); - rte_eth_dev_close(pi); + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + port_flow_flush(pi); + if (rte_eal_process_type() == RTE_PROC_PRIMARY) + rte_eth_dev_close(pi); } remove_invalid_ports(); @@ -3089,7 +3120,7 @@ pmd_test_exit(void) } } for (i = 0 ; i < RTE_DIM(mempools) ; i++) { - if (mempools[i]) + if (rte_eal_process_type() == RTE_PROC_PRIMARY && mempools[i]) rte_mempool_free(mempools[i]); } @@ -3537,6 +3568,10 @@ init_port_dcb_config(portid_t pid, int retval; uint16_t i; + if (num_procs > 1) { + printf("The multi-process feature doesn't support dcb.\n"); + return -ENOTSUP; + } rte_port = &ports[pid]; memset(&port_conf, 0, sizeof(struct rte_eth_conf)); @@ -3635,13 +3670,6 @@ init_port(void) } static void -force_quit(void) -{ - pmd_test_exit(); - prompt_exit(); -} - -static void print_stats(void) { uint8_t i; @@ -3672,12 +3700,16 @@ signal_handler(int signum) if (latencystats_enabled != 0) rte_latencystats_uninit(); #endif - force_quit(); /* Set flag to indicate the force termination. */ f_quit = 1; - /* exit with the expected status */ - signal(signum, SIG_DFL); - kill(getpid(), signum); + if (interactive == 1) { + dup2(testpmd_cl->s_in, testpmd_fd_copy); + close(testpmd_cl->s_in); + } else { + dup2(0, testpmd_fd_copy); + close(0); + } + } } @@ -3702,10 +3734,6 @@ main(int argc, char** argv) rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", rte_strerror(rte_errno)); - if (rte_eal_process_type() == RTE_PROC_SECONDARY) - rte_exit(EXIT_FAILURE, - "Secondary process type not supported.\n"); - ret = register_eth_event_callback(); if (ret != 0) rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); @@ -3801,8 +3829,10 @@ main(int argc, char** argv) } } - if (!no_device_start && start_port(RTE_PORT_ALL) != 0) + if (!no_device_start && start_port(RTE_PORT_ALL) != 0) { + pmd_test_exit(); rte_exit(EXIT_FAILURE, "Start ports failed\n"); + } /* set all ports to promiscuous mode by default */ RTE_ETH_FOREACH_DEV(port_id) { @@ -3848,6 +3878,8 @@ main(int argc, char** argv) } prompt(); pmd_test_exit(); + if (unlikely(f_quit == 1)) + prompt_exit(); } else #endif { @@ -3883,6 +3915,11 @@ main(int argc, char** argv) printf("Press enter to exit\n"); rc = read(0, &c, 1); pmd_test_exit(); + if (unlikely(f_quit == 1)) { + dup2(testpmd_fd_copy, 0); + close(testpmd_fd_copy); + prompt_exit(); + } if (rc < 0) return 1; } diff --git a/app/test-pmd/testpmd.h b/app/test-pmd/testpmd.h index 5f23162..8629c57 100644 --- a/app/test-pmd/testpmd.h +++ b/app/test-pmd/testpmd.h @@ -13,6 +13,7 @@ #include #include #include +#include "cmdline.h" #define RTE_PORT_ALL (~(portid_t)0x0) @@ -24,6 +25,10 @@ #define RTE_PORT_CLOSED (uint16_t)2 #define RTE_PORT_HANDLING (uint16_t)3 +uint8_t f_quit; +int testpmd_fd_copy; +struct cmdline *testpmd_cl; + /* * It is used to allocate the memory for hash key. * The hash key size is NIC dependent. @@ -421,6 +426,8 @@ extern uint64_t noisy_lkup_mem_sz; extern uint64_t noisy_lkup_num_writes; extern uint64_t noisy_lkup_num_reads; extern uint64_t noisy_lkup_num_reads_writes; +extern int proc_id; +extern unsigned num_procs; extern uint8_t dcb_config; extern uint8_t dcb_test;