Patch Detail

HTTP 200 OK
Allow: GET, PUT, PATCH, HEAD, OPTIONS
Content-Type: application/json
Vary: Accept

{
    "id": 117600,
    "url": "http://patchwork.dpdk.org/api/patches/117600/?format=api",
    "web_url": "http://patchwork.dpdk.org/project/dpdk/patch/20221007174336.54354-20-andrew.boyer@amd.com/",
    "project": {
        "id": 1,
        "url": "http://patchwork.dpdk.org/api/projects/1/?format=api",
        "name": "DPDK",
        "link_name": "dpdk",
        "list_id": "dev.dpdk.org",
        "list_email": "dev@dpdk.org",
        "web_url": "http://core.dpdk.org",
        "scm_url": "git://dpdk.org/dpdk",
        "webscm_url": "http://git.dpdk.org/dpdk",
        "list_archive_url": "https://inbox.dpdk.org/dev",
        "list_archive_url_format": "https://inbox.dpdk.org/dev/{}",
        "commit_url_format": ""
    },
    "msgid": "<20221007174336.54354-20-andrew.boyer@amd.com>",
    "list_archive_url": "https://inbox.dpdk.org/dev/20221007174336.54354-20-andrew.boyer@amd.com",
    "date": "2022-10-07T17:43:20",
    "name": "[19/35] net/ionic: overhaul receive side for performance",
    "commit_ref": null,
    "pull_url": null,
    "state": "superseded",
    "archived": true,
    "hash": "451fa6cd740372cae49765d97e2a3f3d7c339578",
    "submitter": {
        "id": 2861,
        "url": "http://patchwork.dpdk.org/api/people/2861/?format=api",
        "name": "Andrew Boyer",
        "email": "Andrew.Boyer@amd.com"
    },
    "delegate": {
        "id": 319,
        "url": "http://patchwork.dpdk.org/api/users/319/?format=api",
        "username": "fyigit",
        "first_name": "Ferruh",
        "last_name": "Yigit",
        "email": "ferruh.yigit@amd.com"
    },
    "mbox": "http://patchwork.dpdk.org/project/dpdk/patch/20221007174336.54354-20-andrew.boyer@amd.com/mbox/",
    "series": [
        {
            "id": 25037,
            "url": "http://patchwork.dpdk.org/api/series/25037/?format=api",
            "web_url": "http://patchwork.dpdk.org/project/dpdk/list/?series=25037",
            "date": "2022-10-07T17:43:01",
            "name": "net/ionic: updates for 22.11 release",
            "version": 1,
            "mbox": "http://patchwork.dpdk.org/series/25037/mbox/"
        }
    ],
    "comments": "http://patchwork.dpdk.org/api/patches/117600/comments/",
    "check": "success",
    "checks": "http://patchwork.dpdk.org/api/patches/117600/checks/",
    "tags": {},
    "related": [],
    "headers": {
        "Return-Path": "<dev-bounces@dpdk.org>",
        "X-Original-To": "patchwork@inbox.dpdk.org",
        "Delivered-To": "patchwork@inbox.dpdk.org",
        "Received": [
            "from mails.dpdk.org (mails.dpdk.org [217.70.189.124])\n\tby inbox.dpdk.org (Postfix) with ESMTP id A2F2FA04FD;\n\tFri,  7 Oct 2022 19:46:37 +0200 (CEST)",
            "from [217.70.189.124] (localhost [127.0.0.1])\n\tby mails.dpdk.org (Postfix) with ESMTP id D67CF42BD0;\n\tFri,  7 Oct 2022 19:44:54 +0200 (CEST)",
            "from NAM02-SN1-obe.outbound.protection.outlook.com\n (mail-sn1anam02on2067.outbound.protection.outlook.com [40.107.96.67])\n by mails.dpdk.org (Postfix) with ESMTP id 6C87F42BB5\n for <dev@dpdk.org>; Fri,  7 Oct 2022 19:44:52 +0200 (CEST)",
            "from MW4PR04CA0265.namprd04.prod.outlook.com (2603:10b6:303:88::30)\n by DM8PR12MB5479.namprd12.prod.outlook.com (2603:10b6:8:38::18) with\n Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5676.20; Fri, 7 Oct\n 2022 17:44:50 +0000",
            "from CO1NAM11FT003.eop-nam11.prod.protection.outlook.com\n (2603:10b6:303:88:cafe::6c) by MW4PR04CA0265.outlook.office365.com\n (2603:10b6:303:88::30) with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.20.5676.26 via Frontend\n Transport; Fri, 7 Oct 2022 17:44:50 +0000",
            "from SATLEXMB04.amd.com (165.204.84.17) by\n CO1NAM11FT003.mail.protection.outlook.com (10.13.175.93) with Microsoft SMTP\n Server (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id\n 15.20.5709.10 via Frontend Transport; Fri, 7 Oct 2022 17:44:49 +0000",
            "from driver-dev1.pensando.io (10.180.168.240) by SATLEXMB04.amd.com\n (10.181.40.145) with Microsoft SMTP Server (version=TLS1_2,\n cipher=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256) id 15.1.2375.28; Fri, 7 Oct\n 2022 12:44:48 -0500"
        ],
        "ARC-Seal": "i=1; a=rsa-sha256; s=arcselector9901; d=microsoft.com; cv=none;\n b=RtKmmQVdnz4HigaD2lnpsVze78OQ6vfkIhUhBP7QszWjA89cHgJ+4n9LYAGpWtDX45KUOpUUhrsELXXuRundMJmB3SaFUdO5vc+hZL4XDzD74USgwHmlq8QLisaAbZcFp0mqabxG5aHlCUlDaArZiDT9obaYXKpmnFGwu28X+5FqDGNWQlEerxlU4t3SpYBmWrYQaThGcULdgEKl9Od3Td40EkCL3MIx9vXYWukgt+l9cAJEK082if7dhndZA6Vik7LVprcdQz4p32EkNpB1niXIQXhFurdQoKvArHGFVG+c7gdwMSTFK0hxgNBpb9ETzxq0j0pgTwW3NPi0mSYO+A==",
        "ARC-Message-Signature": "i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com;\n s=arcselector9901;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-AntiSpam-MessageData-ChunkCount:X-MS-Exchange-AntiSpam-MessageData-0:X-MS-Exchange-AntiSpam-MessageData-1;\n bh=Jwo8lXAAq1UQT1AYjXudvz8kLhFXvZVDdRJU/gGTu0w=;\n b=klq077eiOxfqmWnh1AMDE/B9W9y4CNZTUM/YWSQ7j7dBcydp6rO9u2ZYx9TGXrCGISD/sQDCl5OjnBISuuWh5t+Gi2H3ThV/4aXe1my0GZe5eQGNY1cgU1U5GE5yvxqDwKdzX2W1vZky2BKjtjxInWMsvV2JtIvhKe7aidM8dyX6AiFUtMxJLRhJTD/XlrWYEH4tA05zEehvshcxEFM6f4+PWYx0WU1YxI3Yt4fpw1+1PDApIUgEleWp4vZwE9QQRBMfS0+RofA10fq/Ea1BmgVe7EBFgXxJae8NTXEWtfFGegv+jr9FIqTLyfEpf8X7r8QGQIrjZM7/u6wUbW958g==",
        "ARC-Authentication-Results": "i=1; mx.microsoft.com 1; spf=pass (sender ip is\n 165.204.84.17) smtp.rcpttodomain=dpdk.org smtp.mailfrom=amd.com; dmarc=pass\n (p=quarantine sp=quarantine pct=100) action=none header.from=amd.com;\n dkim=none (message not signed); arc=none",
        "DKIM-Signature": "v=1; a=rsa-sha256; c=relaxed/relaxed; d=amd.com; s=selector1;\n h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck;\n bh=Jwo8lXAAq1UQT1AYjXudvz8kLhFXvZVDdRJU/gGTu0w=;\n b=M6xfDuYseSvPpQX1gFTPgGnDkZrdopdbchyR2c4HeHAbfc2f5RJgc7kh97FJOz0kyLFzAAaDafpTG2cewvoig1GnhBwvMKEe3SAZ4mmhnBQeYWZeh5pzgLz6kn1gHfMxBNj2+VpC7kSSXuaxyD38YGoMezKf1RwODDVGhqHEgN4=",
        "X-MS-Exchange-Authentication-Results": "spf=pass (sender IP is 165.204.84.17)\n smtp.mailfrom=amd.com; dkim=none (message not signed)\n header.d=none;dmarc=pass action=none header.from=amd.com;",
        "Received-SPF": "Pass (protection.outlook.com: domain of amd.com designates\n 165.204.84.17 as permitted sender) receiver=protection.outlook.com;\n client-ip=165.204.84.17; helo=SATLEXMB04.amd.com; pr=C",
        "From": "Andrew Boyer <andrew.boyer@amd.com>",
        "To": "<dev@dpdk.org>",
        "CC": "Andrew Boyer <andrew.boyer@amd.com>, Neel Patel <neel.patel@amd.com>",
        "Subject": "[PATCH 19/35] net/ionic: overhaul receive side for performance",
        "Date": "Fri, 7 Oct 2022 10:43:20 -0700",
        "Message-ID": "<20221007174336.54354-20-andrew.boyer@amd.com>",
        "X-Mailer": "git-send-email 2.17.1",
        "In-Reply-To": "<20221007174336.54354-1-andrew.boyer@amd.com>",
        "References": "<20221007174336.54354-1-andrew.boyer@amd.com>",
        "MIME-Version": "1.0",
        "Content-Type": "text/plain",
        "X-Originating-IP": "[10.180.168.240]",
        "X-ClientProxiedBy": "SATLEXMB04.amd.com (10.181.40.145) To SATLEXMB04.amd.com\n (10.181.40.145)",
        "X-EOPAttributedMessage": "0",
        "X-MS-PublicTrafficType": "Email",
        "X-MS-TrafficTypeDiagnostic": "CO1NAM11FT003:EE_|DM8PR12MB5479:EE_",
        "X-MS-Office365-Filtering-Correlation-Id": "ba62d562-6e16-4925-179c-08daa88ba211",
        "X-MS-Exchange-SenderADCheck": "1",
        "X-MS-Exchange-AntiSpam-Relay": "0",
        "X-Microsoft-Antispam": "BCL:0;",
        "X-Microsoft-Antispam-Message-Info": "\n upNXFNTPLQfMvU1jtrLPh+jTfEJULtvtz5+0Gs5OBKyL64NEuPspVssdnHtb9R5l3y3YvoAeMtbandbvcUI3rUGNhEPUif4+xBAROthUBu5OZP8fhUw2sewDZVaZZsocg/LTNQ0nt6gvsIfjJucycZwhgMd7yqyUlFgA/WgYI1b+/gZLOanIVbIqxe/S8Be/fBOWoLtKkxKlemBRF4m2X73JZHTufBzwUzYexJ6vPzB238tQwtaW1zqf0+fjuvRHh9SG+bQZRrMnUPhsWR1sd4EY6w3ymm/m2jKsuKdtIutwnG8WFm2FFVKzRJOZ9EPJqoZYtccf09eyiqFxqFOpcqP6lVel3rfv9mI6HMnIMVMrGqRKfpJX+pRhmB66xb7HcUW/2rKpwfLn4vDzd03wnWDa7Ndyp2jm98JcjQ+jEqfKCSbueg9fz7f3jw28/rJOohjnbR/cJb9BqjJb3yGp4AHZMh0Ws1xTQshvp6P+ltanC0b8ynrq1CxjaHEYD24UvcneBktFkvvX/hiXG7sEsDOgMasLDRr5nV4PftITXfqVzIhPm7i7qiXjyTCFtM6OCKyprfQhsaUFr86P8cGp7zI6HG9PxFl+xUp0clBm1pqFFSByv0xegk4aQ9R9jfErsgksYztjzxi5hvJQPcigWFuvvz6gZHULOQHr4KoOsTPrnwdR4J7vbHpTQOBJQVGOINmuOFJOar5QPZU4RedWgb2mYiX8fi1/HjnoJ8BZUKU9A9J+kH/bH3wfd8EC6GhQBWQC1GySgZA60XfsgC2wNVSJfQwq1v+2oHw/i9mhK7lfmrAvPdmfAci9CVGWIDbH",
        "X-Forefront-Antispam-Report": "CIP:165.204.84.17; CTRY:US; LANG:en; SCL:1; SRV:;\n IPV:CAL; SFV:NSPM; H:SATLEXMB04.amd.com; PTR:InfoDomainNonexistent; CAT:NONE;\n SFS:(13230022)(4636009)(39860400002)(376002)(396003)(346002)(136003)(451199015)(46966006)(40470700004)(36840700001)(86362001)(356005)(478600001)(44832011)(81166007)(54906003)(30864003)(2906002)(82310400005)(6916009)(2616005)(336012)(36756003)(186003)(1076003)(5660300002)(82740400003)(70586007)(26005)(4326008)(70206006)(8936002)(316002)(8676002)(426003)(47076005)(16526019)(41300700001)(6666004)(40460700003)(40480700001)(83380400001)(36860700001)(36900700001);\n DIR:OUT; SFP:1101;",
        "X-OriginatorOrg": "amd.com",
        "X-MS-Exchange-CrossTenant-OriginalArrivalTime": "07 Oct 2022 17:44:49.7733 (UTC)",
        "X-MS-Exchange-CrossTenant-Network-Message-Id": "\n ba62d562-6e16-4925-179c-08daa88ba211",
        "X-MS-Exchange-CrossTenant-Id": "3dd8961f-e488-4e60-8e11-a82d994e183d",
        "X-MS-Exchange-CrossTenant-OriginalAttributedTenantConnectingIp": "\n TenantId=3dd8961f-e488-4e60-8e11-a82d994e183d; Ip=[165.204.84.17];\n Helo=[SATLEXMB04.amd.com]",
        "X-MS-Exchange-CrossTenant-AuthSource": "\n CO1NAM11FT003.eop-nam11.prod.protection.outlook.com",
        "X-MS-Exchange-CrossTenant-AuthAs": "Anonymous",
        "X-MS-Exchange-CrossTenant-FromEntityHeader": "HybridOnPrem",
        "X-MS-Exchange-Transport-CrossTenantHeadersStamped": "DM8PR12MB5479",
        "X-BeenThere": "dev@dpdk.org",
        "X-Mailman-Version": "2.1.29",
        "Precedence": "list",
        "List-Id": "DPDK patches and discussions <dev.dpdk.org>",
        "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n <mailto:dev-request@dpdk.org?subject=unsubscribe>",
        "List-Archive": "<http://mails.dpdk.org/archives/dev/>",
        "List-Post": "<mailto:dev@dpdk.org>",
        "List-Help": "<mailto:dev-request@dpdk.org?subject=help>",
        "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n <mailto:dev-request@dpdk.org?subject=subscribe>",
        "Errors-To": "dev-bounces@dpdk.org"
    },
    "content": "Linearize RX mbuf chains in the expanded info array.\nClean one and fill one per CQE (completions are not coalesced).\nTouch the mbufs as little as possible in the fill stage.\nWhen touching the mbuf in the clean stage, use the rearm_data unions.\nRing the doorbell once at the end of the bulk clean/fill.\n\nSigned-off-by: Neel Patel <neel.patel@amd.com>\nSigned-off-by: Andrew Boyer <andrew.boyer@amd.com>\n---\n doc/guides/rel_notes/release_22_11.rst |   1 +\n drivers/net/ionic/ionic_dev.h          |   2 +-\n drivers/net/ionic/ionic_lif.c          |  49 +++++-\n drivers/net/ionic/ionic_lif.h          |   3 +-\n drivers/net/ionic/ionic_rxtx.c         | 225 ++++++++++++-------------\n drivers/net/ionic/ionic_rxtx.h         |   1 -\n 6 files changed, 159 insertions(+), 122 deletions(-)",
    "diff": "diff --git a/doc/guides/rel_notes/release_22_11.rst b/doc/guides/rel_notes/release_22_11.rst\nindex 552cc5b62c..d7eced510e 100644\n--- a/doc/guides/rel_notes/release_22_11.rst\n+++ b/doc/guides/rel_notes/release_22_11.rst\n@@ -83,6 +83,7 @@ New Features\n   Updated the ionic PMD with new features and improvements, including:\n \n   * Updated to reflect that Pensando has been acquired by AMD.\n+  * Enhanced data path to provide substantial performance improvements.\n \n Removed Items\n -------------\ndiff --git a/drivers/net/ionic/ionic_dev.h b/drivers/net/ionic/ionic_dev.h\nindex 55a9485bff..6a80ebc71b 100644\n--- a/drivers/net/ionic/ionic_dev.h\n+++ b/drivers/net/ionic/ionic_dev.h\n@@ -132,7 +132,7 @@ struct ionic_dev {\n #define Q_NEXT_TO_POST(_q, _n)\t(((_q)->head_idx + (_n)) & ((_q)->size_mask))\n #define Q_NEXT_TO_SRVC(_q, _n)\t(((_q)->tail_idx + (_n)) & ((_q)->size_mask))\n \n-#define IONIC_INFO_IDX(_q, _i)\t(_i)\n+#define IONIC_INFO_IDX(_q, _i)\t((_i) * (_q)->num_segs)\n #define IONIC_INFO_PTR(_q, _i)\t(&(_q)->info[IONIC_INFO_IDX((_q), _i)])\n \n struct ionic_queue {\ndiff --git a/drivers/net/ionic/ionic_lif.c b/drivers/net/ionic/ionic_lif.c\nindex cc64aedaa1..db5d42dda6 100644\n--- a/drivers/net/ionic/ionic_lif.c\n+++ b/drivers/net/ionic/ionic_lif.c\n@@ -116,7 +116,6 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats\n \t\tstruct ionic_rx_stats *rx_stats = &lif->rxqcqs[i]->stats;\n \t\tstats->ierrors +=\n \t\t\trx_stats->bad_cq_status +\n-\t\t\trx_stats->no_room +\n \t\t\trx_stats->bad_len;\n \t}\n \n@@ -136,7 +135,6 @@ ionic_lif_get_abs_stats(const struct ionic_lif *lif, struct rte_eth_stats *stats\n \t\tstats->q_ibytes[i] = rx_stats->bytes;\n \t\tstats->q_errors[i] =\n \t\t\trx_stats->bad_cq_status +\n-\t\t\trx_stats->no_room +\n \t\t\trx_stats->bad_len;\n \t}\n \n@@ -608,8 +606,9 @@ ionic_qcq_alloc(struct ionic_lif *lif,\n \n \tnew->lif = lif;\n \n+\t/* Most queue types will store 1 ptr per descriptor */\n \tnew->q.info = rte_calloc_socket(\"ionic\",\n-\t\t\t\tnum_descs, sizeof(void *),\n+\t\t\t\tnum_descs * num_segs, sizeof(void *),\n \t\t\t\trte_mem_page_size(), socket_id);\n \tif (!new->q.info) {\n \t\tIONIC_PRINT(ERR, \"Cannot allocate queue info\");\n@@ -698,6 +697,42 @@ ionic_qcq_free(struct ionic_qcq *qcq)\n \trte_free(qcq);\n }\n \n+static uint64_t\n+ionic_rx_rearm_data(struct ionic_lif *lif)\n+{\n+\tstruct rte_mbuf rxm;\n+\n+\tmemset(&rxm, 0, sizeof(rxm));\n+\n+\trte_mbuf_refcnt_set(&rxm, 1);\n+\trxm.data_off = RTE_PKTMBUF_HEADROOM;\n+\trxm.nb_segs = 1;\n+\trxm.port = lif->port_id;\n+\n+\trte_compiler_barrier();\n+\n+\tRTE_BUILD_BUG_ON(sizeof(rxm.rearm_data[0]) != sizeof(uint64_t));\n+\treturn rxm.rearm_data[0];\n+}\n+\n+static uint64_t\n+ionic_rx_seg_rearm_data(struct ionic_lif *lif)\n+{\n+\tstruct rte_mbuf rxm;\n+\n+\tmemset(&rxm, 0, sizeof(rxm));\n+\n+\trte_mbuf_refcnt_set(&rxm, 1);\n+\trxm.data_off = 0;  /* no headroom */\n+\trxm.nb_segs = 1;\n+\trxm.port = lif->port_id;\n+\n+\trte_compiler_barrier();\n+\n+\tRTE_BUILD_BUG_ON(sizeof(rxm.rearm_data[0]) != sizeof(uint64_t));\n+\treturn rxm.rearm_data[0];\n+}\n+\n int\n ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index,\n \t\tuint16_t nrxq_descs, struct rte_mempool *mb_pool,\n@@ -721,11 +756,13 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index,\n \n \t/*\n \t * Calculate how many fragment pointers might be stored in queue.\n+\t * This is the worst-case number, so that there's enough room in\n+\t * the info array.\n \t */\n \tmax_segs = 1 + (max_mtu + RTE_PKTMBUF_HEADROOM - 1) / seg_size;\n \n-\tIONIC_PRINT(DEBUG, \"rxq %u frame_size %u seg_size %u max_segs %u\",\n-\t\tindex, lif->frame_size, seg_size, max_segs);\n+\tIONIC_PRINT(DEBUG, \"rxq %u max_mtu %u seg_size %u max_segs %u\",\n+\t\tindex, max_mtu, seg_size, max_segs);\n \tif (max_segs > max_segs_fw) {\n \t\tIONIC_PRINT(ERR, \"Rx mbuf size insufficient (%d > %d avail)\",\n \t\t\tmax_segs, max_segs_fw);\n@@ -751,6 +788,8 @@ ionic_rx_qcq_alloc(struct ionic_lif *lif, uint32_t socket_id, uint32_t index,\n \trxq->flags = flags;\n \trxq->seg_size = seg_size;\n \trxq->hdr_seg_size = hdr_seg_size;\n+\trxq->rearm_data = ionic_rx_rearm_data(lif);\n+\trxq->rearm_seg_data = ionic_rx_seg_rearm_data(lif);\n \n \tlif->rxqcqs[index] = rxq;\n \t*rxq_out = rxq;\ndiff --git a/drivers/net/ionic/ionic_lif.h b/drivers/net/ionic/ionic_lif.h\nindex 237fd0a2ef..b0bd721b06 100644\n--- a/drivers/net/ionic/ionic_lif.h\n+++ b/drivers/net/ionic/ionic_lif.h\n@@ -40,7 +40,6 @@ struct ionic_rx_stats {\n \tuint64_t packets;\n \tuint64_t bytes;\n \tuint64_t bad_cq_status;\n-\tuint64_t no_room;\n \tuint64_t bad_len;\n \tuint64_t mtods;\n };\n@@ -80,6 +79,8 @@ struct ionic_rx_qcq {\n \n \t/* cacheline2 */\n \tstruct rte_mempool *mb_pool;\n+\tuint64_t rearm_data;\n+\tuint64_t rearm_seg_data;\n \tuint16_t frame_size;\t/* Based on configured MTU */\n \tuint16_t hdr_seg_size;\t/* Length of first segment of RX chain */\n \tuint16_t seg_size;\t/* Length of all subsequent segments */\ndiff --git a/drivers/net/ionic/ionic_rxtx.c b/drivers/net/ionic/ionic_rxtx.c\nindex 2a34465e46..bb6ca019d9 100644\n--- a/drivers/net/ionic/ionic_rxtx.c\n+++ b/drivers/net/ionic/ionic_rxtx.c\n@@ -72,7 +72,11 @@ ionic_rx_empty(struct ionic_rx_qcq *rxq)\n {\n \tstruct ionic_queue *q = &rxq->qcq.q;\n \n-\tionic_empty_array(q->info, q->num_descs, 0);\n+\t/*\n+\t * Walk the full info array so that the clean up includes any\n+\t * fragments that were left dangling for later reuse\n+\t */\n+\tionic_empty_array(q->info, q->num_descs * q->num_segs, 0);\n }\n \n /*********************************************************************\n@@ -658,9 +662,6 @@ ionic_prep_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts)\n  *\n  **********************************************************************/\n \n-static void ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,\n-\t\tstruct rte_mbuf *mbuf);\n-\n void\n ionic_rxq_info_get(struct rte_eth_dev *dev, uint16_t queue_id,\n \t\tstruct rte_eth_rxq_info *qinfo)\n@@ -763,64 +764,67 @@ ionic_dev_rx_queue_setup(struct rte_eth_dev *eth_dev,\n \treturn 0;\n }\n \n+/*\n+ * Cleans one descriptor. Connects the filled mbufs into a chain.\n+ * Does not advance the tail index.\n+ */\n static __rte_always_inline void\n-ionic_rx_clean(struct ionic_rx_qcq *rxq,\n-\t\tuint32_t q_desc_index, uint32_t cq_desc_index,\n+ionic_rx_clean_one(struct ionic_rx_qcq *rxq,\n+\t\tstruct ionic_rxq_comp *cq_desc,\n \t\tstruct ionic_rx_service *rx_svc)\n {\n \tstruct ionic_queue *q = &rxq->qcq.q;\n-\tstruct ionic_cq *cq = &rxq->qcq.cq;\n-\tstruct ionic_rxq_comp *cq_desc_base = cq->base;\n-\tstruct ionic_rxq_comp *cq_desc = &cq_desc_base[cq_desc_index];\n-\tstruct rte_mbuf *rxm, *rxm_seg;\n+\tstruct rte_mbuf *rxm, *rxm_seg, *prev_rxm;\n+\tstruct ionic_rx_stats *stats = &rxq->stats;\n \tuint64_t pkt_flags = 0;\n \tuint32_t pkt_type;\n-\tstruct ionic_rx_stats *stats = &rxq->stats;\n-\tuint32_t left;\n+\tuint32_t left, i;\n+\tuint16_t cq_desc_len;\n \tvoid **info;\n \n-\tassert(q_desc_index == cq_desc->comp_index);\n+\tcq_desc_len = rte_le_to_cpu_16(cq_desc->len);\n \n-\tinfo = IONIC_INFO_PTR(q, cq_desc->comp_index);\n+\tinfo = IONIC_INFO_PTR(q, q->tail_idx);\n \n \trxm = info[0];\n \n \tif (cq_desc->status) {\n \t\tstats->bad_cq_status++;\n-\t\tionic_rx_recycle(q, q_desc_index, rxm);\n-\t\treturn;\n-\t}\n-\n-\tif (rx_svc->nb_rx >= rx_svc->nb_pkts) {\n-\t\tstats->no_room++;\n-\t\tionic_rx_recycle(q, q_desc_index, rxm);\n \t\treturn;\n \t}\n \n-\tif (cq_desc->len > rxq->frame_size || cq_desc->len == 0) {\n+\tif (cq_desc_len > rxq->frame_size || cq_desc_len == 0) {\n \t\tstats->bad_len++;\n-\t\tionic_rx_recycle(q, q_desc_index, rxm);\n \t\treturn;\n \t}\n \n-\trxm->data_off = RTE_PKTMBUF_HEADROOM;\n-\trte_prefetch1((char *)rxm->buf_addr + rxm->data_off);\n-\trxm->nb_segs = 1; /* cq_desc->num_sg_elems */\n-\trxm->pkt_len = cq_desc->len;\n-\trxm->port = rxq->qcq.lif->port_id;\n+\tinfo[0] = NULL;\n \n-\trxm->data_len = RTE_MIN(rxq->hdr_seg_size, cq_desc->len);\n-\tleft = cq_desc->len - rxm->data_len;\n+\t/* Set the mbuf metadata based on the cq entry */\n+\trxm->rearm_data[0] = rxq->rearm_data;\n+\trxm->pkt_len = cq_desc_len;\n+\trxm->data_len = RTE_MIN(rxq->hdr_seg_size, cq_desc_len);\n+\tleft = cq_desc_len - rxm->data_len;\n+\trxm->nb_segs = cq_desc->num_sg_elems + 1;\n+\tprev_rxm = rxm;\n \n-\trxm_seg = rxm->next;\n-\twhile (rxm_seg && left) {\n+\tfor (i = 1; i < rxm->nb_segs && left; i++) {\n+\t\trxm_seg = info[i];\n+\t\tinfo[i] = NULL;\n+\n+\t\t/* Set the chained mbuf metadata */\n+\t\trxm_seg->rearm_data[0] = rxq->rearm_seg_data;\n \t\trxm_seg->data_len = RTE_MIN(rxq->seg_size, left);\n \t\tleft -= rxm_seg->data_len;\n \n-\t\trxm_seg = rxm_seg->next;\n-\t\trxm->nb_segs++;\n+\t\t/* Link the mbuf */\n+\t\tprev_rxm->next = rxm_seg;\n+\t\tprev_rxm = rxm_seg;\n \t}\n \n+\t/* Terminate the mbuf chain */\n+\tprev_rxm->next = NULL;\n+\n \t/* RSS */\n \tpkt_flags |= RTE_MBUF_F_RX_RSS_HASH;\n \trxm->hash.rss = rte_le_to_cpu_32(cq_desc->rss_hash);\n@@ -897,77 +901,74 @@ ionic_rx_clean(struct ionic_rx_qcq *rxq,\n \tstats->bytes += rxm->pkt_len;\n }\n \n-static void\n-ionic_rx_recycle(struct ionic_queue *q, uint32_t q_desc_index,\n-\t\t struct rte_mbuf *mbuf)\n-{\n-\tstruct ionic_rxq_desc *desc_base = q->base;\n-\tstruct ionic_rxq_desc *old = &desc_base[q_desc_index];\n-\tstruct ionic_rxq_desc *new = &desc_base[q->head_idx];\n-\n-\tnew->addr = old->addr;\n-\tnew->len = old->len;\n-\n-\tq->info[q->head_idx] = mbuf;\n-\n-\tq->head_idx = Q_NEXT_TO_POST(q, 1);\n-\n-\tionic_q_flush(q);\n-}\n-\n+/*\n+ * Fills one descriptor with mbufs. Does not advance the head index.\n+ */\n static __rte_always_inline int\n-ionic_rx_fill(struct ionic_rx_qcq *rxq)\n+ionic_rx_fill_one(struct ionic_rx_qcq *rxq)\n {\n \tstruct ionic_queue *q = &rxq->qcq.q;\n+\tstruct rte_mbuf *rxm, *rxm_seg;\n \tstruct ionic_rxq_desc *desc, *desc_base = q->base;\n \tstruct ionic_rxq_sg_desc *sg_desc, *sg_desc_base = q->sg_base;\n-\tstruct ionic_rxq_sg_elem *elem;\n+\trte_iova_t data_iova;\n+\tuint32_t i;\n \tvoid **info;\n-\trte_iova_t dma_addr;\n-\tuint32_t i, j;\n \n-\t/* Initialize software ring entries */\n-\tfor (i = ionic_q_space_avail(q); i; i--) {\n-\t\tstruct rte_mbuf *rxm = rte_mbuf_raw_alloc(rxq->mb_pool);\n-\t\tstruct rte_mbuf *prev_rxm_seg;\n+\tinfo = IONIC_INFO_PTR(q, q->head_idx);\n+\tdesc = &desc_base[q->head_idx];\n+\tsg_desc = &sg_desc_base[q->head_idx];\n+\n+\t/* mbuf is unused => whole chain is unused */\n+\tif (unlikely(info[0]))\n+\t\treturn 0;\n+\n+\trxm = rte_mbuf_raw_alloc(rxq->mb_pool);\n+\tif (unlikely(rxm == NULL)) {\n+\t\tassert(0);\n+\t\treturn -ENOMEM;\n+\t}\n+\n+\tinfo[0] = rxm;\n+\n+\tdata_iova = rte_mbuf_data_iova_default(rxm);\n+\tdesc->addr = rte_cpu_to_le_64(data_iova);\n \n-\t\tif (rxm == NULL) {\n-\t\t\tIONIC_PRINT(ERR, \"RX mbuf alloc failed\");\n+\tfor (i = 1; i < q->num_segs; i++) {\n+\t\t/* mbuf is unused => rest of the chain is unused */\n+\t\tif (info[i])\n+\t\t\treturn 0;\n+\n+\t\trxm_seg = rte_mbuf_raw_alloc(rxq->mb_pool);\n+\t\tif (rxm_seg == NULL) {\n+\t\t\tassert(0);\n \t\t\treturn -ENOMEM;\n \t\t}\n \n-\t\tinfo = IONIC_INFO_PTR(q, q->head_idx);\n+\t\tinfo[i] = rxm_seg;\n \n-\t\tdesc = &desc_base[q->head_idx];\n-\t\tdma_addr = rte_cpu_to_le_64(rte_mbuf_data_iova_default(rxm));\n-\t\tdesc->addr = dma_addr;\n-\t\trxm->next = NULL;\n-\n-\t\tprev_rxm_seg = rxm;\n-\t\tsg_desc = &sg_desc_base[q->head_idx];\n-\t\telem = sg_desc->elems;\n-\t\tfor (j = 0; j < q->num_segs - 1u; j++) {\n-\t\t\tstruct rte_mbuf *rxm_seg;\n-\t\t\trte_iova_t data_iova;\n-\n-\t\t\trxm_seg = rte_mbuf_raw_alloc(rxq->mb_pool);\n-\t\t\tif (rxm_seg == NULL) {\n-\t\t\t\tIONIC_PRINT(ERR, \"RX mbuf alloc failed\");\n-\t\t\t\treturn -ENOMEM;\n-\t\t\t}\n+\t\t/* The data_off does not get set to 0 until later */\n+\t\tdata_iova = rxm_seg->buf_iova;\n+\t\tsg_desc->elems[i - 1].addr = rte_cpu_to_le_64(data_iova);\n+\t}\n \n-\t\t\trxm_seg->data_off = 0;\n-\t\t\tdata_iova = rte_mbuf_data_iova(rxm_seg);\n-\t\t\tdma_addr = rte_cpu_to_le_64(data_iova);\n-\t\t\telem->addr = dma_addr;\n-\t\t\telem++;\n+\treturn 0;\n+}\n \n-\t\t\trxm_seg->next = NULL;\n-\t\t\tprev_rxm_seg->next = rxm_seg;\n-\t\t\tprev_rxm_seg = rxm_seg;\n-\t\t}\n+/*\n+ * Fills all descriptors with mbufs.\n+ */\n+static int __rte_cold\n+ionic_rx_fill(struct ionic_rx_qcq *rxq)\n+{\n+\tstruct ionic_queue *q = &rxq->qcq.q;\n+\tuint32_t i;\n+\tint err;\n \n-\t\tinfo[0] = rxm;\n+\tfor (i = 1; i < q->num_descs; i++) {\n+\t\terr = ionic_rx_fill_one(rxq);\n+\t\tif (err)\n+\t\t\treturn err;\n \n \t\tq->head_idx = Q_NEXT_TO_POST(q, 1);\n \t}\n@@ -1056,53 +1057,52 @@ ionic_dev_rx_queue_start(struct rte_eth_dev *eth_dev, uint16_t rx_queue_id)\n \treturn 0;\n }\n \n+/*\n+ * Walk the CQ to find completed receive descriptors.\n+ * Any completed descriptor found is refilled.\n+ */\n static __rte_always_inline void\n ionic_rxq_service(struct ionic_rx_qcq *rxq, uint32_t work_to_do,\n \t\tstruct ionic_rx_service *rx_svc)\n {\n \tstruct ionic_cq *cq = &rxq->qcq.cq;\n \tstruct ionic_queue *q = &rxq->qcq.q;\n+\tstruct ionic_rxq_desc *q_desc_base = q->base;\n \tstruct ionic_rxq_comp *cq_desc, *cq_desc_base = cq->base;\n-\tbool more;\n-\tuint32_t curr_q_tail_idx, curr_cq_tail_idx;\n \tuint32_t work_done = 0;\n \n-\tif (work_to_do == 0)\n-\t\treturn;\n-\n \tcq_desc = &cq_desc_base[cq->tail_idx];\n+\n \twhile (color_match(cq_desc->pkt_type_color, cq->done_color)) {\n-\t\tcurr_cq_tail_idx = cq->tail_idx;\n \t\tcq->tail_idx = Q_NEXT_TO_SRVC(cq, 1);\n \n \t\tif (cq->tail_idx == 0)\n \t\t\tcq->done_color = !cq->done_color;\n \n-\t\t/* Prefetch the next 4 descriptors */\n-\t\tif ((cq->tail_idx & 0x3) == 0)\n-\t\t\trte_prefetch0(&cq_desc_base[cq->tail_idx]);\n-\n-\t\tdo {\n-\t\t\tmore = (q->tail_idx != cq_desc->comp_index);\n+\t\t/* Prefetch 8 x 8B bufinfo */\n+\t\trte_prefetch0(IONIC_INFO_PTR(q, Q_NEXT_TO_SRVC(q, 8)));\n+\t\t/* Prefetch 4 x 16B comp */\n+\t\trte_prefetch0(&cq_desc_base[Q_NEXT_TO_SRVC(cq, 4)]);\n+\t\t/* Prefetch 4 x 16B descriptors */\n+\t\trte_prefetch0(&q_desc_base[Q_NEXT_TO_POST(q, 4)]);\n \n-\t\t\tcurr_q_tail_idx = q->tail_idx;\n-\t\t\tq->tail_idx = Q_NEXT_TO_SRVC(q, 1);\n+\t\tionic_rx_clean_one(rxq, cq_desc, rx_svc);\n \n-\t\t\t/* Prefetch the next 4 descriptors */\n-\t\t\tif ((q->tail_idx & 0x3) == 0)\n-\t\t\t\t/* q desc info */\n-\t\t\t\trte_prefetch0(&q->info[q->tail_idx]);\n+\t\tq->tail_idx = Q_NEXT_TO_SRVC(q, 1);\n \n-\t\t\tionic_rx_clean(rxq, curr_q_tail_idx, curr_cq_tail_idx,\n-\t\t\t\trx_svc);\n+\t\t(void)ionic_rx_fill_one(rxq);\n \n-\t\t} while (more);\n+\t\tq->head_idx = Q_NEXT_TO_POST(q, 1);\n \n \t\tif (++work_done == work_to_do)\n \t\t\tbreak;\n \n \t\tcq_desc = &cq_desc_base[cq->tail_idx];\n \t}\n+\n+\t/* Update the queue indices and ring the doorbell */\n+\tif (work_done)\n+\t\tionic_q_flush(q);\n }\n \n /*\n@@ -1141,12 +1141,9 @@ ionic_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts,\n \tstruct ionic_rx_service rx_svc;\n \n \trx_svc.rx_pkts = rx_pkts;\n-\trx_svc.nb_pkts = nb_pkts;\n \trx_svc.nb_rx = 0;\n \n \tionic_rxq_service(rxq, nb_pkts, &rx_svc);\n \n-\tionic_rx_fill(rxq);\n-\n \treturn rx_svc.nb_rx;\n }\ndiff --git a/drivers/net/ionic/ionic_rxtx.h b/drivers/net/ionic/ionic_rxtx.h\nindex 91a9073803..79ec1112de 100644\n--- a/drivers/net/ionic/ionic_rxtx.h\n+++ b/drivers/net/ionic/ionic_rxtx.h\n@@ -10,7 +10,6 @@\n struct ionic_rx_service {\n \t/* cb in */\n \tstruct rte_mbuf **rx_pkts;\n-\tuint16_t nb_pkts;\n \t/* cb out */\n \tuint16_t nb_rx;\n };\n",
    "prefixes": [
        "19/35"
    ]
}