From patchwork Wed Oct 9 14:16:53 2019 Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit X-Patchwork-Submitter: "Wei Hu (Xavier)" X-Patchwork-Id: 60805 X-Patchwork-Delegate: ferruh.yigit@amd.com Return-Path: X-Original-To: patchwork@dpdk.org Delivered-To: patchwork@dpdk.org Received: from [92.243.14.124] (localhost [127.0.0.1]) by dpdk.org (Postfix) with ESMTP id 80FAF1E8D7; Wed, 9 Oct 2019 16:17:21 +0200 (CEST) Received: from smtp.tom.com (smtprz14.163.net [106.3.154.247]) by dpdk.org (Postfix) with ESMTP id EA9101E545 for ; Wed, 9 Oct 2019 16:17:11 +0200 (CEST) Received: from my-app01.tom.com (my-app01.tom.com [127.0.0.1]) by freemail01.tom.com (Postfix) with ESMTP id EE9261C82ADF for ; Wed, 9 Oct 2019 22:17:27 +0800 (CST) Received: from my-app01.tom.com (HELO smtp.tom.com) ([127.0.0.1]) by my-app01 (TOM SMTP Server) with SMTP ID 608605570 for ; Wed, 09 Oct 2019 22:17:27 +0800 (CST) Received: from antispam1.tom.com (unknown [172.25.16.55]) by freemail01.tom.com (Postfix) with ESMTP id E4D161C82956 for ; Wed, 9 Oct 2019 22:17:27 +0800 (CST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/simple; d=tom.com; s=201807; t=1570630647; bh=aJUSDkdq0EnHqibkV2LcMmB1C5h+K0+dXVi6lDMHG2A=; h=From:To:Cc:Subject:Date:In-Reply-To:References:From; b=Swi+iJHjltA1T69+1s2WrvGMXm6PFfOfSN8xm/qO3Cnys1a3s73LyiTkSryrDlOPQ ZuSIwUWkMkQYGgWTqzQmoUHSP+YFiPA+1DZaQM9jSQzRjzZuqzAtAYJ/686fXCFdVT 9Yq5JlZoFz0OnkTdx6s/k9NWzRz6xldEIvKY+62k= Received: from antispam1.tom.com (antispam1.tom.com [127.0.0.1]) by antispam1.tom.com (Postfix) with ESMTP id 7B32710016C0 for ; Wed, 9 Oct 2019 22:16:01 +0800 (CST) X-Virus-Scanned: Debian amavisd-new at antispam1.tom.com Received: from antispam1.tom.com ([127.0.0.1]) by antispam1.tom.com (antispam1.tom.com [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id If4dyj4FnOYC for ; Wed, 9 Oct 2019 22:16:00 +0800 (CST) Received: from localhost.localdomain (unknown [114.119.4.74]) by antispam1.tom.com (Postfix) with ESMTPA id 7C2151001667; Wed, 9 Oct 2019 22:16:00 +0800 (CST) From: "Wei Hu (Xavier)" To: dev@dpdk.org Cc: xavier.huwei@huawei.com Date: Wed, 9 Oct 2019 22:16:53 +0800 Message-Id: <20191009141653.39364-5-xavier.huwei@tom.com> X-Mailer: git-send-email 2.23.0 In-Reply-To: <20191009141653.39364-1-xavier.huwei@tom.com> References: <20191009141653.39364-1-xavier.huwei@tom.com> MIME-Version: 1.0 Subject: [dpdk-dev] [PATCH 4/4] net/hns3: restores bus_master_en and msix_enable during PF FLR reset X-BeenThere: dev@dpdk.org X-Mailman-Version: 2.1.15 Precedence: list List-Id: DPDK patches and discussions List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: dev-bounces@dpdk.org Sender: "dev" From: Chunsong Feng PF FLR resets the PCIe ECAM space of all VFs under the PF and does not automatically recover. Therefore, the VF driver needs to restore the ECAM configuration, including bus_master_en, msix_enable. Signed-off-by: Chunsong Feng Signed-off-by: Wei Hu (Xavier) --- drivers/net/hns3/hns3_cmd.c | 15 ---- drivers/net/hns3/hns3_ethdev.c | 5 ++ drivers/net/hns3/hns3_ethdev_vf.c | 135 +++++++++++++++++++++++++++++- drivers/net/hns3/hns3_intr.c | 5 +- 4 files changed, 139 insertions(+), 21 deletions(-) diff --git a/drivers/net/hns3/hns3_cmd.c b/drivers/net/hns3/hns3_cmd.c index 58776c2ec..65a5af8e4 100644 --- a/drivers/net/hns3/hns3_cmd.c +++ b/drivers/net/hns3/hns3_cmd.c @@ -216,24 +216,9 @@ hns3_cmd_csq_clean(struct hns3_hw *hw) if (!is_valid_csq_clean_head(csq, head)) { struct hns3_adapter *hns = HNS3_DEV_HW_TO_ADAPTER(hw); - uint32_t global; - uint32_t fun_rst; hns3_err(hw, "wrong cmd head (%u, %u-%u)", head, csq->next_to_use, csq->next_to_clean); rte_atomic16_set(&hw->reset.disable_cmd, 1); - if (hns->is_vf) { - global = hns3_read_dev(hw, HNS3_VF_RST_ING); - fun_rst = hns3_read_dev(hw, HNS3_FUN_RST_ING); - hns3_err(hw, "Delayed VF reset global: %x fun_rst: %x", - global, fun_rst); - hns3_atomic_set_bit(HNS3_VF_RESET, &hw->reset.pending); - } else { - global = hns3_read_dev(hw, HNS3_GLOBAL_RESET_REG); - fun_rst = hns3_read_dev(hw, HNS3_FUN_RST_ING); - hns3_err(hw, "Delayed IMP reset global: %x fun_rst: %x", - global, fun_rst); - hns3_atomic_set_bit(HNS3_IMP_RESET, &hw->reset.pending); - } hns3_schedule_delayed_reset(hns); diff --git a/drivers/net/hns3/hns3_ethdev.c b/drivers/net/hns3/hns3_ethdev.c index 862a717fd..3435bce26 100644 --- a/drivers/net/hns3/hns3_ethdev.c +++ b/drivers/net/hns3/hns3_ethdev.c @@ -4680,6 +4680,11 @@ hns3_reset_service(void *param) rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_REQUESTED); hns3_err(hw, "Handling interrupts in delayed tasks"); hns3_interrupt_handler(&rte_eth_devices[hw->data->port_id]); + reset_level = hns3_get_reset_level(hns, &hw->reset.pending); + if (reset_level == HNS3_NONE_RESET) { + hns3_err(hw, "No reset level is set, try IMP reset"); + hns3_atomic_set_bit(HNS3_IMP_RESET, &hw->reset.pending); + } } rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_NONE); diff --git a/drivers/net/hns3/hns3_ethdev_vf.c b/drivers/net/hns3/hns3_ethdev_vf.c index 121beb58d..1e9acd4f9 100644 --- a/drivers/net/hns3/hns3_ethdev_vf.c +++ b/drivers/net/hns3/hns3_ethdev_vf.c @@ -9,6 +9,8 @@ #include #include #include +#include + #include #include #include @@ -24,6 +26,7 @@ #include #include #include +#include #include "hns3_ethdev.h" #include "hns3_logs.h" @@ -56,6 +59,81 @@ static enum hns3_reset_level hns3vf_get_reset_level(struct hns3_hw *hw, static int hns3vf_dev_mtu_set(struct rte_eth_dev *dev, uint16_t mtu); static int hns3vf_dev_configure_vlan(struct rte_eth_dev *dev); +/* set PCI bus mastering */ +static void +hns3vf_set_bus_master(const struct rte_pci_device *device, bool op) +{ + uint16_t reg; + + rte_pci_read_config(device, ®, sizeof(reg), PCI_COMMAND); + + if (op) + /* set the master bit */ + reg |= PCI_COMMAND_MASTER; + else + reg &= ~(PCI_COMMAND_MASTER); + + rte_pci_write_config(device, ®, sizeof(reg), PCI_COMMAND); +} + +/** + * hns3vf_find_pci_capability - lookup a capability in the PCI capability list + * @cap: the capability + * + * Return the address of the given capability within the PCI capability list. + */ +static inline int +hns3vf_find_pci_capability(const struct rte_pci_device *device, int cap) +{ +#define MAX_PCIE_CAPABILITY 48 + uint16_t status; + uint8_t pos; + uint8_t id; + int ttl; + + rte_pci_read_config(device, &status, sizeof(status), PCI_STATUS); + if (!(status & PCI_STATUS_CAP_LIST)) + return 0; + + ttl = MAX_PCIE_CAPABILITY; + rte_pci_read_config(device, &pos, sizeof(pos), PCI_CAPABILITY_LIST); + while (ttl-- && pos >= PCI_STD_HEADER_SIZEOF) { + rte_pci_read_config(device, &id, sizeof(id), + (pos + PCI_CAP_LIST_ID)); + + if (id == 0xFF) + break; + + if (id == cap) + return (int)pos; + + rte_pci_read_config(device, &pos, sizeof(pos), + (pos + PCI_CAP_LIST_NEXT)); + } + return 0; +} + +static int +hns3vf_enable_msix(const struct rte_pci_device *device, bool op) +{ + uint16_t control; + int pos; + + pos = hns3vf_find_pci_capability(device, PCI_CAP_ID_MSIX); + if (pos) { + rte_pci_read_config(device, &control, sizeof(control), + (pos + PCI_MSIX_FLAGS)); + if (op) + control |= PCI_MSIX_FLAGS_ENABLE; + else + control &= ~PCI_MSIX_FLAGS_ENABLE; + rte_pci_write_config(device, &control, sizeof(control), + (pos + PCI_MSIX_FLAGS)); + return 0; + } + return -1; +} + static int hns3vf_add_mac_addr(struct rte_eth_dev *dev, struct rte_ether_addr *mac_addr, __attribute__ ((unused)) uint32_t idx, @@ -1308,9 +1386,30 @@ hns3vf_wait_hardware_ready(struct hns3_adapter *hns) struct hns3_wait_data *wait_data = hw->reset.wait_data; struct timeval tv; - if (wait_data->result == HNS3_WAIT_SUCCESS) - return 0; - else if (wait_data->result == HNS3_WAIT_TIMEOUT) { + if (wait_data->result == HNS3_WAIT_SUCCESS) { + /* + * After vf reset is ready, the PF may not have completed + * the reset processing. The vf sending mbox to PF may fail + * during the pf reset, so it is better to add extra delay. + */ + if (hw->reset.level == HNS3_VF_FUNC_RESET || + hw->reset.level == HNS3_FLR_RESET) + return 0; + /* Reset retry process, no need to add extra delay. */ + if (hw->reset.attempts) + return 0; + if (wait_data->check_completion == NULL) + return 0; + + wait_data->check_completion = NULL; + wait_data->interval = 1 * MSEC_PER_SEC * USEC_PER_MSEC; + wait_data->count = 1; + wait_data->result = HNS3_WAIT_REQUEST; + rte_eal_alarm_set(wait_data->interval, hns3_wait_callback, + wait_data); + hns3_warn(hw, "hardware is ready, delay 1 sec for PF reset complete"); + return -EAGAIN; + } else if (wait_data->result == HNS3_WAIT_TIMEOUT) { gettimeofday(&tv, NULL); hns3_warn(hw, "Reset step4 hardware not ready after reset time=%ld.%.6ld", tv.tv_sec, tv.tv_usec); @@ -1473,6 +1572,11 @@ hns3vf_reset_service(void *param) rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_REQUESTED); hns3_err(hw, "Handling interrupts in delayed tasks"); hns3vf_interrupt_handler(&rte_eth_devices[hw->data->port_id]); + reset_level = hns3vf_get_reset_level(hw, &hw->reset.pending); + if (reset_level == HNS3_NONE_RESET) { + hns3_err(hw, "No reset level is set, try global reset"); + hns3_atomic_set_bit(HNS3_VF_RESET, &hw->reset.pending); + } } rte_atomic16_set(&hns->hw.reset.schedule, SCHEDULE_NONE); @@ -1498,14 +1602,35 @@ hns3vf_reset_service(void *param) static int hns3vf_reinit_dev(struct hns3_adapter *hns) { + struct rte_eth_dev *eth_dev = &rte_eth_devices[hns->hw.data->port_id]; + struct rte_pci_device *pci_dev = RTE_ETH_DEV_TO_PCI(eth_dev); struct hns3_hw *hw = &hns->hw; int ret; + if (hw->reset.level == HNS3_VF_FULL_RESET) { + rte_intr_disable(&pci_dev->intr_handle); + hns3vf_set_bus_master(pci_dev, true); + } + /* Firmware command initialize */ ret = hns3_cmd_init(hw); if (ret) { hns3_err(hw, "Failed to init cmd: %d", ret); - return ret; + goto err_cmd_init; + } + + if (hw->reset.level == HNS3_VF_FULL_RESET) { + /* + * UIO enables msix by writing the pcie configuration space + * vfio_pci enables msix in rte_intr_enable. + */ + if (pci_dev->kdrv == RTE_KDRV_IGB_UIO || + pci_dev->kdrv == RTE_KDRV_UIO_GENERIC) { + if (hns3vf_enable_msix(pci_dev, true)) + hns3_err(hw, "Failed to enable msix"); + } + + rte_intr_enable(&pci_dev->intr_handle); } ret = hns3_reset_all_queues(hns); @@ -1522,6 +1647,8 @@ hns3vf_reinit_dev(struct hns3_adapter *hns) return 0; +err_cmd_init: + hns3vf_set_bus_master(pci_dev, false); err_init: hns3_cmd_uninit(hw); return ret; diff --git a/drivers/net/hns3/hns3_intr.c b/drivers/net/hns3/hns3_intr.c index 9e2d81156..6c3ebd3ee 100644 --- a/drivers/net/hns3/hns3_intr.c +++ b/drivers/net/hns3/hns3_intr.c @@ -890,11 +890,12 @@ hns3_reset_err_handle(struct hns3_adapter *hns) hns3_warn(hw, "%s reset fail fail_cnt:%" PRIx64 " success_cnt:%" PRIx64 " global_cnt:%" PRIx64 " imp_cnt:%" PRIx64 " request_cnt:%" PRIx64 " exec_cnt:%" PRIx64 - " merge_cnt:%" PRIx64, + " merge_cnt:%" PRIx64 "adapter_state:%d", reset_string[hw->reset.level], hw->reset.stats.fail_cnt, hw->reset.stats.success_cnt, hw->reset.stats.global_cnt, hw->reset.stats.imp_cnt, hw->reset.stats.request_cnt, - hw->reset.stats.exec_cnt, hw->reset.stats.merge_cnt); + hw->reset.stats.exec_cnt, hw->reset.stats.merge_cnt, + hw->adapter_state); /* IMP no longer waiting the ready flag */ hns3_notify_reset_ready(hw, true);