Patch Detail
get:
Show a patch.
patch:
Update a patch.
put:
Update a patch.
GET /api/patches/46540/?format=api
{ "id": 46540, "url": "http://patchwork.dpdk.org/api/patches/46540/?format=api", "web_url": "http://patchwork.dpdk.org/project/dpdk/patch/1539233972-49860-6-git-send-email-honnappa.nagarahalli@arm.com/", "project": { "id": 1, "url": "http://patchwork.dpdk.org/api/projects/1/?format=api", "name": "DPDK", "link_name": "dpdk", "list_id": "dev.dpdk.org", "list_email": "dev@dpdk.org", "web_url": "http://core.dpdk.org", "scm_url": "git://dpdk.org/dpdk", "webscm_url": "http://git.dpdk.org/dpdk", "list_archive_url": "https://inbox.dpdk.org/dev", "list_archive_url_format": "https://inbox.dpdk.org/dev/{}", "commit_url_format": "" }, "msgid": "<1539233972-49860-6-git-send-email-honnappa.nagarahalli@arm.com>", "list_archive_url": "https://inbox.dpdk.org/dev/1539233972-49860-6-git-send-email-honnappa.nagarahalli@arm.com", "date": "2018-10-11T04:59:30", "name": "[v2,5/7] hash: fix rw concurrency while moving keys", "commit_ref": null, "pull_url": null, "state": "superseded", "archived": true, "hash": "60ac888ac8c9cb60d6c37c793903ba75e00e0335", "submitter": { "id": 1045, "url": "http://patchwork.dpdk.org/api/people/1045/?format=api", "name": "Honnappa Nagarahalli", "email": "honnappa.nagarahalli@arm.com" }, "delegate": { "id": 1, "url": "http://patchwork.dpdk.org/api/users/1/?format=api", "username": "tmonjalo", "first_name": "Thomas", "last_name": "Monjalon", "email": "thomas@monjalon.net" }, "mbox": "http://patchwork.dpdk.org/project/dpdk/patch/1539233972-49860-6-git-send-email-honnappa.nagarahalli@arm.com/mbox/", "series": [ { "id": 1824, "url": "http://patchwork.dpdk.org/api/series/1824/?format=api", "web_url": "http://patchwork.dpdk.org/project/dpdk/list/?series=1824", "date": "2018-10-11T04:59:25", "name": "Address reader-writer concurrency in rte_hash", "version": 2, "mbox": "http://patchwork.dpdk.org/series/1824/mbox/" } ], "comments": "http://patchwork.dpdk.org/api/patches/46540/comments/", "check": "fail", "checks": "http://patchwork.dpdk.org/api/patches/46540/checks/", "tags": {}, "related": [], "headers": { "Return-Path": "<dev-bounces@dpdk.org>", "X-Original-To": "patchwork@dpdk.org", "Delivered-To": "patchwork@dpdk.org", "Received": [ "from [92.243.14.124] (localhost [127.0.0.1])\n\tby dpdk.org (Postfix) with ESMTP id 0AE4D1B439;\n\tThu, 11 Oct 2018 06:59:57 +0200 (CEST)", "from foss.arm.com (foss.arm.com [217.140.101.70])\n\tby dpdk.org (Postfix) with ESMTP id 62D611B3B8\n\tfor <dev@dpdk.org>; Thu, 11 Oct 2018 06:59:48 +0200 (CEST)", "from usa-sjc-imap-foss1.foss.arm.com (unknown [10.72.51.249])\n\tby usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id D4D567A9;\n\tWed, 10 Oct 2018 21:59:47 -0700 (PDT)", "from 2p2660v4-1.austin.arm.com (2p2660v4-1.austin.arm.com\n\t[10.118.12.190])\n\tby usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPSA id\n\t7043D3F5B3; Wed, 10 Oct 2018 21:59:47 -0700 (PDT)" ], "From": "Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>", "To": "bruce.richardson@intel.com,\n\tpablo.de.lara.guarch@intel.com", "Cc": "dev@dpdk.org, yipeng1.wang@intel.com, honnappa.nagarahalli@arm.com,\n\tDharmik.Thakkar@arm.com, nd@arm.com", "Date": "Wed, 10 Oct 2018 23:59:30 -0500", "Message-Id": "<1539233972-49860-6-git-send-email-honnappa.nagarahalli@arm.com>", "X-Mailer": "git-send-email 2.7.4", "In-Reply-To": "<1539233972-49860-1-git-send-email-honnappa.nagarahalli@arm.com>", "References": "<1539233972-49860-1-git-send-email-honnappa.nagarahalli@arm.com>", "Subject": "[dpdk-dev] [PATCH v2 5/7] hash: fix rw concurrency while moving keys", "X-BeenThere": "dev@dpdk.org", "X-Mailman-Version": "2.1.15", "Precedence": "list", "List-Id": "DPDK patches and discussions <dev.dpdk.org>", "List-Unsubscribe": "<https://mails.dpdk.org/options/dev>,\n\t<mailto:dev-request@dpdk.org?subject=unsubscribe>", "List-Archive": "<http://mails.dpdk.org/archives/dev/>", "List-Post": "<mailto:dev@dpdk.org>", "List-Help": "<mailto:dev-request@dpdk.org?subject=help>", "List-Subscribe": "<https://mails.dpdk.org/listinfo/dev>,\n\t<mailto:dev-request@dpdk.org?subject=subscribe>", "Errors-To": "dev-bounces@dpdk.org", "Sender": "\"dev\" <dev-bounces@dpdk.org>" }, "content": "Reader-writer concurrency issue, caused by moving the keys\nto their alternative locations during key insert, is solved\nby introducing a global counter(tbl_chng_cnt) indicating a\nchange in table.\n\nSigned-off-by: Honnappa Nagarahalli <honnappa.nagarahalli@arm.com>\nReviewed-by: Gavin Hu <gavin.hu@arm.com>\nReviewed-by: Ola Liljedahl <ola.liljedahl@arm.com>\nReviewed-by: Steve Capper <steve.capper@arm.com>\nReviewed-by: Yipeng Wang <yipeng1.wang@intel.com>\n---\n lib/librte_hash/rte_cuckoo_hash.c | 306 +++++++++++++++++++++++++-------------\n lib/librte_hash/rte_cuckoo_hash.h | 3 +\n 2 files changed, 209 insertions(+), 100 deletions(-)", "diff": "diff --git a/lib/librte_hash/rte_cuckoo_hash.c b/lib/librte_hash/rte_cuckoo_hash.c\nindex e2b0260..dfd5f2a 100644\n--- a/lib/librte_hash/rte_cuckoo_hash.c\n+++ b/lib/librte_hash/rte_cuckoo_hash.c\n@@ -96,6 +96,7 @@ rte_hash_create(const struct rte_hash_parameters *params)\n \tunsigned int readwrite_concur_support = 0;\n \tunsigned int writer_takes_lock = 0;\n \tunsigned int recycle_on_del = 1;\n+\tuint32_t *tbl_chng_cnt = NULL;\n \n \trte_hash_function default_hash_func = (rte_hash_function)rte_jhash;\n \n@@ -210,6 +211,14 @@ rte_hash_create(const struct rte_hash_parameters *params)\n \t\tgoto err_unlock;\n \t}\n \n+\ttbl_chng_cnt = rte_zmalloc_socket(NULL, sizeof(uint32_t),\n+\t\t\tRTE_CACHE_LINE_SIZE, params->socket_id);\n+\n+\tif (tbl_chng_cnt == NULL) {\n+\t\tRTE_LOG(ERR, HASH, \"memory allocation failed\\n\");\n+\t\tgoto err_unlock;\n+\t}\n+\n /*\n * If x86 architecture is used, select appropriate compare function,\n * which may use x86 intrinsics, otherwise use memcmp\n@@ -276,6 +285,8 @@ rte_hash_create(const struct rte_hash_parameters *params)\n \t\tdefault_hash_func : params->hash_func;\n \th->key_store = k;\n \th->free_slots = r;\n+\th->tbl_chng_cnt = tbl_chng_cnt;\n+\t*h->tbl_chng_cnt = 0;\n \th->hw_trans_mem_support = hw_trans_mem_support;\n \th->multi_writer_support = multi_writer_support;\n \th->readwrite_concur_support = readwrite_concur_support;\n@@ -321,6 +332,7 @@ rte_hash_create(const struct rte_hash_parameters *params)\n \trte_free(h);\n \trte_free(buckets);\n \trte_free(k);\n+\trte_free(tbl_chng_cnt);\n \treturn NULL;\n }\n \n@@ -359,6 +371,7 @@ rte_hash_free(struct rte_hash *h)\n \trte_ring_free(h->free_slots);\n \trte_free(h->key_store);\n \trte_free(h->buckets);\n+\trte_free(h->tbl_chng_cnt);\n \trte_free(h);\n \trte_free(te);\n }\n@@ -456,6 +469,7 @@ rte_hash_reset(struct rte_hash *h)\n \t__hash_rw_writer_lock(h);\n \tmemset(h->buckets, 0, h->num_buckets * sizeof(struct rte_hash_bucket));\n \tmemset(h->key_store, 0, h->key_entry_size * (h->entries + 1));\n+\t*h->tbl_chng_cnt = 0;\n \n \t/* clear the free ring */\n \twhile (rte_ring_dequeue(h->free_slots, &ptr) == 0)\n@@ -650,11 +664,27 @@ rte_hash_cuckoo_move_insert_mw(const struct rte_hash *h,\n \t\tif (unlikely(&h->buckets[prev_alt_bkt_idx]\n \t\t\t\t!= curr_bkt)) {\n \t\t\t/* revert it to empty, otherwise duplicated keys */\n-\t\t\tcurr_bkt->key_idx[curr_slot] = EMPTY_SLOT;\n+\t\t\t__atomic_store_n(&curr_bkt->key_idx[curr_slot],\n+\t\t\t\tEMPTY_SLOT,\n+\t\t\t\t__ATOMIC_RELEASE);\n \t\t\t__hash_rw_writer_unlock(h);\n \t\t\treturn -1;\n \t\t}\n \n+\t\t/* Inform the previous move. The current move need\n+\t\t * not be informed now as the current bucket entry\n+\t\t * is present in both primary and secondary.\n+\t\t * Since there is one writer, load acquires on\n+\t\t * tbl_chng_cnt are not required.\n+\t\t */\n+\t\t__atomic_store_n(h->tbl_chng_cnt,\n+\t\t\t\t *h->tbl_chng_cnt + 1,\n+\t\t\t\t __ATOMIC_RELEASE);\n+\t\t/* The stores to sig_alt and sig_current should not\n+\t\t * move above the store to tbl_chng_cnt.\n+\t\t */\n+\t\t__atomic_thread_fence(__ATOMIC_RELEASE);\n+\n \t\t/* Need to swap current/alt sig to allow later\n \t\t * Cuckoo insert to move elements back to its\n \t\t * primary bucket if available\n@@ -673,6 +703,20 @@ rte_hash_cuckoo_move_insert_mw(const struct rte_hash *h,\n \t\tcurr_bkt = curr_node->bkt;\n \t}\n \n+\t/* Inform the previous move. The current move need\n+\t * not be informed now as the current bucket entry\n+\t * is present in both primary and secondary.\n+\t * Since there is one writer, load acquires on\n+\t * tbl_chng_cnt are not required.\n+\t */\n+\t__atomic_store_n(h->tbl_chng_cnt,\n+\t\t\t *h->tbl_chng_cnt + 1,\n+\t\t\t __ATOMIC_RELEASE);\n+\t/* The stores to sig_alt and sig_current should not\n+\t * move above the store to tbl_chng_cnt.\n+\t */\n+\t__atomic_thread_fence(__ATOMIC_RELEASE);\n+\n \tcurr_bkt->sig_current[curr_slot] = sig;\n \tcurr_bkt->sig_alt[curr_slot] = alt_hash;\n \t/* Release the new bucket entry */\n@@ -937,30 +981,56 @@ __rte_hash_lookup_with_hash(const struct rte_hash *h, const void *key,\n \tuint32_t bucket_idx;\n \thash_sig_t alt_hash;\n \tstruct rte_hash_bucket *bkt;\n+\tuint32_t cnt_b, cnt_a;\n \tint ret;\n \n-\tbucket_idx = sig & h->bucket_bitmask;\n-\tbkt = &h->buckets[bucket_idx];\n-\n \t__hash_rw_reader_lock(h);\n \n-\t/* Check if key is in primary location */\n-\tret = search_one_bucket(h, key, sig, data, bkt);\n-\tif (ret != -1) {\n-\t\t__hash_rw_reader_unlock(h);\n-\t\treturn ret;\n-\t}\n-\t/* Calculate secondary hash */\n-\talt_hash = rte_hash_secondary_hash(sig);\n-\tbucket_idx = alt_hash & h->bucket_bitmask;\n-\tbkt = &h->buckets[bucket_idx];\n+\tdo {\n+\t\t/* Load the table change counter before the lookup\n+\t\t * starts. Acquire semantics will make sure that\n+\t\t * loads in search_one_bucket are not hoisted.\n+\t\t */\n+\t\tcnt_b = __atomic_load_n(h->tbl_chng_cnt,\n+\t\t\t\t__ATOMIC_ACQUIRE);\n+\n+\t\tbucket_idx = sig & h->bucket_bitmask;\n+\t\tbkt = &h->buckets[bucket_idx];\n+\n+\t\t/* Check if key is in primary location */\n+\t\tret = search_one_bucket(h, key, sig, data, bkt);\n+\t\tif (ret != -1) {\n+\t\t\t__hash_rw_reader_unlock(h);\n+\t\t\treturn ret;\n+\t\t}\n+\t\t/* Calculate secondary hash */\n+\t\talt_hash = rte_hash_secondary_hash(sig);\n+\t\tbucket_idx = alt_hash & h->bucket_bitmask;\n+\t\tbkt = &h->buckets[bucket_idx];\n+\n+\t\t/* Check if key is in secondary location */\n+\t\tret = search_one_bucket(h, key, alt_hash, data, bkt);\n+\t\tif (ret != -1) {\n+\t\t\t__hash_rw_reader_unlock(h);\n+\t\t\treturn ret;\n+\t\t}\n+\n+\t\t/* The loads of sig_current in search_one_bucket\n+\t\t * should not move below the load from tbl_chng_cnt.\n+\t\t */\n+\t\t__atomic_thread_fence(__ATOMIC_ACQUIRE);\n+\t\t/* Re-read the table change counter to check if the\n+\t\t * table has changed during search. If yes, re-do\n+\t\t * the search.\n+\t\t * This load should not get hoisted. The load\n+\t\t * acquires on cnt_b, key index in primary bucket\n+\t\t * and key index in secondary bucket will make sure\n+\t\t * that it does not get hoisted.\n+\t\t */\n+\t\tcnt_a = __atomic_load_n(h->tbl_chng_cnt,\n+\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\t} while (cnt_b != cnt_a);\n \n-\t/* Check if key is in secondary location */\n-\tret = search_one_bucket(h, key, alt_hash, data, bkt);\n-\tif (ret != -1) {\n-\t\t__hash_rw_reader_unlock(h);\n-\t\treturn ret;\n-\t}\n \t__hash_rw_reader_unlock(h);\n \treturn -ENOENT;\n }\n@@ -1242,6 +1312,7 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,\n \tuint32_t prim_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n \tuint32_t sec_hitmask[RTE_HASH_LOOKUP_BULK_MAX] = {0};\n \tvoid *pdata[RTE_HASH_LOOKUP_BULK_MAX];\n+\tuint32_t cnt_b, cnt_a;\n \n \t/* Prefetch first keys */\n \tfor (i = 0; i < PREFETCH_OFFSET && i < num_keys; i++)\n@@ -1277,102 +1348,137 @@ __rte_hash_lookup_bulk(const struct rte_hash *h, const void **keys,\n \t}\n \n \t__hash_rw_reader_lock(h);\n-\t/* Compare signatures and prefetch key slot of first hit */\n-\tfor (i = 0; i < num_keys; i++) {\n-\t\tcompare_signatures(&prim_hitmask[i], &sec_hitmask[i],\n+\tdo {\n+\t\t/* Load the table change counter before the lookup\n+\t\t * starts. Acquire semantics will make sure that\n+\t\t * loads in compare_signatures are not hoisted.\n+\t\t */\n+\t\tcnt_b = __atomic_load_n(h->tbl_chng_cnt,\n+\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\n+\t\t/* Compare signatures and prefetch key slot of first hit */\n+\t\tfor (i = 0; i < num_keys; i++) {\n+\t\t\tcompare_signatures(&prim_hitmask[i], &sec_hitmask[i],\n \t\t\t\tprimary_bkt[i], secondary_bkt[i],\n \t\t\t\tprim_hash[i], sec_hash[i], h->sig_cmp_fn);\n \n-\t\tif (prim_hitmask[i]) {\n-\t\t\tuint32_t first_hit = __builtin_ctzl(prim_hitmask[i]);\n-\t\t\tuint32_t key_idx = primary_bkt[i]->key_idx[first_hit];\n-\t\t\tconst struct rte_hash_key *key_slot =\n-\t\t\t\t(const struct rte_hash_key *)(\n-\t\t\t\t(const char *)h->key_store +\n-\t\t\t\tkey_idx * h->key_entry_size);\n-\t\t\trte_prefetch0(key_slot);\n-\t\t\tcontinue;\n-\t\t}\n+\t\t\tif (prim_hitmask[i]) {\n+\t\t\t\tuint32_t first_hit =\n+\t\t\t\t\t\t__builtin_ctzl(prim_hitmask[i]);\n+\t\t\t\tuint32_t key_idx =\n+\t\t\t\t\tprimary_bkt[i]->key_idx[first_hit];\n+\t\t\t\tconst struct rte_hash_key *key_slot =\n+\t\t\t\t\t(const struct rte_hash_key *)(\n+\t\t\t\t\t(const char *)h->key_store +\n+\t\t\t\t\tkey_idx * h->key_entry_size);\n+\t\t\t\trte_prefetch0(key_slot);\n+\t\t\t\tcontinue;\n+\t\t\t}\n \n-\t\tif (sec_hitmask[i]) {\n-\t\t\tuint32_t first_hit = __builtin_ctzl(sec_hitmask[i]);\n-\t\t\tuint32_t key_idx = secondary_bkt[i]->key_idx[first_hit];\n-\t\t\tconst struct rte_hash_key *key_slot =\n-\t\t\t\t(const struct rte_hash_key *)(\n-\t\t\t\t(const char *)h->key_store +\n-\t\t\t\tkey_idx * h->key_entry_size);\n-\t\t\trte_prefetch0(key_slot);\n+\t\t\tif (sec_hitmask[i]) {\n+\t\t\t\tuint32_t first_hit =\n+\t\t\t\t\t\t__builtin_ctzl(sec_hitmask[i]);\n+\t\t\t\tuint32_t key_idx =\n+\t\t\t\t\tsecondary_bkt[i]->key_idx[first_hit];\n+\t\t\t\tconst struct rte_hash_key *key_slot =\n+\t\t\t\t\t(const struct rte_hash_key *)(\n+\t\t\t\t\t(const char *)h->key_store +\n+\t\t\t\t\tkey_idx * h->key_entry_size);\n+\t\t\t\trte_prefetch0(key_slot);\n+\t\t\t}\n \t\t}\n-\t}\n \n-\t/* Compare keys, first hits in primary first */\n-\tfor (i = 0; i < num_keys; i++) {\n-\t\tpositions[i] = -ENOENT;\n-\t\twhile (prim_hitmask[i]) {\n-\t\t\tuint32_t hit_index = __builtin_ctzl(prim_hitmask[i]);\n+\t\t/* Compare keys, first hits in primary first */\n+\t\tfor (i = 0; i < num_keys; i++) {\n+\t\t\tpositions[i] = -ENOENT;\n+\t\t\twhile (prim_hitmask[i]) {\n+\t\t\t\tuint32_t hit_index =\n+\t\t\t\t\t\t__builtin_ctzl(prim_hitmask[i]);\n \n-\t\t\tuint32_t key_idx =\n-\t\t\t__atomic_load_n(\n-\t\t\t\t&primary_bkt[i]->key_idx[hit_index],\n-\t\t\t\t__ATOMIC_ACQUIRE);\n-\t\t\tconst struct rte_hash_key *key_slot =\n-\t\t\t\t(const struct rte_hash_key *)(\n-\t\t\t\t(const char *)h->key_store +\n-\t\t\t\tkey_idx * h->key_entry_size);\n-\n-\t\t\tif (key_idx != EMPTY_SLOT)\n-\t\t\t\tpdata[i] = __atomic_load_n(&key_slot->pdata,\n-\t\t\t\t\t\t__ATOMIC_ACQUIRE);\n-\t\t\t/*\n-\t\t\t * If key index is 0, do not compare key,\n-\t\t\t * as it is checking the dummy slot\n-\t\t\t */\n-\t\t\tif (!!key_idx & !rte_hash_cmp_eq(key_slot->key, keys[i], h)) {\n-\t\t\t\tif (data != NULL)\n-\t\t\t\t\tdata[i] = pdata[i];\n+\t\t\t\tuint32_t key_idx =\n+\t\t\t\t__atomic_load_n(\n+\t\t\t\t\t&primary_bkt[i]->key_idx[hit_index],\n+\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\t\t\t\tconst struct rte_hash_key *key_slot =\n+\t\t\t\t\t(const struct rte_hash_key *)(\n+\t\t\t\t\t(const char *)h->key_store +\n+\t\t\t\t\tkey_idx * h->key_entry_size);\n \n-\t\t\t\thits |= 1ULL << i;\n-\t\t\t\tpositions[i] = key_idx - 1;\n-\t\t\t\tgoto next_key;\n+\t\t\t\tif (key_idx != EMPTY_SLOT)\n+\t\t\t\t\tpdata[i] = __atomic_load_n(\n+\t\t\t\t\t\t\t&key_slot->pdata,\n+\t\t\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\t\t\t\t/*\n+\t\t\t\t * If key index is 0, do not compare key,\n+\t\t\t\t * as it is checking the dummy slot\n+\t\t\t\t */\n+\t\t\t\tif (!!key_idx &\n+\t\t\t\t\t!rte_hash_cmp_eq(\n+\t\t\t\t\t\tkey_slot->key, keys[i], h)) {\n+\t\t\t\t\tif (data != NULL)\n+\t\t\t\t\t\tdata[i] = pdata[i];\n+\n+\t\t\t\t\thits |= 1ULL << i;\n+\t\t\t\t\tpositions[i] = key_idx - 1;\n+\t\t\t\t\tgoto next_key;\n+\t\t\t\t}\n+\t\t\t\tprim_hitmask[i] &= ~(1 << (hit_index));\n \t\t\t}\n-\t\t\tprim_hitmask[i] &= ~(1 << (hit_index));\n-\t\t}\n \n-\t\twhile (sec_hitmask[i]) {\n-\t\t\tuint32_t hit_index = __builtin_ctzl(sec_hitmask[i]);\n+\t\t\twhile (sec_hitmask[i]) {\n+\t\t\t\tuint32_t hit_index =\n+\t\t\t\t\t\t__builtin_ctzl(sec_hitmask[i]);\n \n-\t\t\tuint32_t key_idx =\n-\t\t\t__atomic_load_n(\n-\t\t\t\t&secondary_bkt[i]->key_idx[hit_index],\n-\t\t\t\t__ATOMIC_ACQUIRE);\n-\t\t\tconst struct rte_hash_key *key_slot =\n-\t\t\t\t(const struct rte_hash_key *)(\n-\t\t\t\t(const char *)h->key_store +\n-\t\t\t\tkey_idx * h->key_entry_size);\n-\n-\t\t\tif (key_idx != EMPTY_SLOT)\n-\t\t\t\tpdata[i] = __atomic_load_n(&key_slot->pdata,\n-\t\t\t\t\t\t__ATOMIC_ACQUIRE);\n-\n-\t\t\t/*\n-\t\t\t * If key index is 0, do not compare key,\n-\t\t\t * as it is checking the dummy slot\n-\t\t\t */\n+\t\t\t\tuint32_t key_idx =\n+\t\t\t\t__atomic_load_n(\n+\t\t\t\t\t&secondary_bkt[i]->key_idx[hit_index],\n+\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\t\t\t\tconst struct rte_hash_key *key_slot =\n+\t\t\t\t\t(const struct rte_hash_key *)(\n+\t\t\t\t\t(const char *)h->key_store +\n+\t\t\t\t\tkey_idx * h->key_entry_size);\n \n-\t\t\tif (!!key_idx & !rte_hash_cmp_eq(key_slot->key, keys[i], h)) {\n-\t\t\t\tif (data != NULL)\n-\t\t\t\t\tdata[i] = pdata[i];\n+\t\t\t\tif (key_idx != EMPTY_SLOT)\n+\t\t\t\t\tpdata[i] = __atomic_load_n(\n+\t\t\t\t\t\t\t&key_slot->pdata,\n+\t\t\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\t\t\t\t/*\n+\t\t\t\t * If key index is 0, do not compare key,\n+\t\t\t\t * as it is checking the dummy slot\n+\t\t\t\t */\n \n-\t\t\t\thits |= 1ULL << i;\n-\t\t\t\tpositions[i] = key_idx - 1;\n-\t\t\t\tgoto next_key;\n+\t\t\t\tif (!!key_idx &\n+\t\t\t\t\t!rte_hash_cmp_eq(\n+\t\t\t\t\t\tkey_slot->key, keys[i], h)) {\n+\t\t\t\t\tif (data != NULL)\n+\t\t\t\t\t\tdata[i] = pdata[i];\n+\n+\t\t\t\t\thits |= 1ULL << i;\n+\t\t\t\t\tpositions[i] = key_idx - 1;\n+\t\t\t\t\tgoto next_key;\n+\t\t\t\t}\n+\t\t\t\tsec_hitmask[i] &= ~(1 << (hit_index));\n \t\t\t}\n-\t\t\tsec_hitmask[i] &= ~(1 << (hit_index));\n-\t\t}\n \n next_key:\n-\t\tcontinue;\n-\t}\n+\t\t\tcontinue;\n+\t\t}\n+\n+\t\t/* The loads of sig_current in compare_signatures\n+\t\t * should not move below the load from tbl_chng_cnt.\n+\t\t */\n+\t\t__atomic_thread_fence(__ATOMIC_ACQUIRE);\n+\t\t/* Re-read the table change counter to check if the\n+\t\t * table has changed during search. If yes, re-do\n+\t\t * the search.\n+\t\t * This load should not get hoisted. The load\n+\t\t * acquires on cnt_b, primary key index and secondary\n+\t\t * key index will make sure that it does not get\n+\t\t * hoisted.\n+\t\t */\n+\t\tcnt_a = __atomic_load_n(h->tbl_chng_cnt,\n+\t\t\t\t\t__ATOMIC_ACQUIRE);\n+\t} while (cnt_b != cnt_a);\n \n \t__hash_rw_reader_unlock(h);\n \ndiff --git a/lib/librte_hash/rte_cuckoo_hash.h b/lib/librte_hash/rte_cuckoo_hash.h\nindex a44c6be..cf50ada 100644\n--- a/lib/librte_hash/rte_cuckoo_hash.h\n+++ b/lib/librte_hash/rte_cuckoo_hash.h\n@@ -1,5 +1,6 @@\n /* SPDX-License-Identifier: BSD-3-Clause\n * Copyright(c) 2016 Intel Corporation\n+ * Copyright(c) 2018 Arm Limited\n */\n \n /* rte_cuckoo_hash.h\n@@ -196,6 +197,8 @@ struct rte_hash {\n \t * to the key table.\n \t */\n \trte_rwlock_t *readwrite_lock; /**< Read-write lock thread-safety. */\n+\tuint32_t *tbl_chng_cnt;\n+\t/**< Indicates if the hash table changed from last read. */\n } __rte_cache_aligned;\n \n struct queue_node {\n", "prefixes": [ "v2", "5/7" ] }