@@ -37,4 +37,6 @@ DEPDIRS-linuxapp := common
DIRS-$(CONFIG_RTE_EXEC_ENV_BSDAPP) += bsdapp
DEPDIRS-bsdapp := common
+LDLIBS += -lnuma
+
include $(RTE_SDK)/mk/rte.subdir.mk
@@ -54,6 +54,7 @@
#include <sys/time.h>
#include <signal.h>
#include <setjmp.h>
+#include <numaif.h>
#include <rte_log.h>
#include <rte_memory.h>
@@ -358,6 +359,19 @@ static int huge_wrap_sigsetjmp(void)
return sigsetjmp(huge_jmpenv, 1);
}
+#ifndef ULONG_SIZE
+#define ULONG_SIZE sizeof(unsigned long)
+#endif
+#ifndef ULONG_BITS
+#define ULONG_BITS (ULONG_SIZE * CHAR_BIT)
+#endif
+#ifndef DIV_ROUND_UP
+#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
+#endif
+#ifndef BITS_TO_LONGS
+#define BITS_TO_LONGS(nr) DIV_ROUND_UP(nr, ULONG_SIZE)
+#endif
+
/*
* Mmap all hugepages of hugepage table: it first open a file in
* hugetlbfs, then mmap() hugepage_sz data in it. If orig is set, the
@@ -366,18 +380,78 @@ static int huge_wrap_sigsetjmp(void)
* map continguous physical blocks in contiguous virtual blocks.
*/
static unsigned
-map_all_hugepages(struct hugepage_file *hugepg_tbl,
- struct hugepage_info *hpi, int orig)
+map_all_hugepages(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi,
+ uint64_t *essential_memory, int orig)
{
int fd;
unsigned i;
void *virtaddr;
void *vma_addr = NULL;
size_t vma_len = 0;
+ unsigned long nodemask[BITS_TO_LONGS(RTE_MAX_NUMA_NODES)] = {0UL};
+ unsigned long maxnode = 0;
+ int node_id = -1;
+ bool numa_available = true;
+
+ /* Check if kernel supports NUMA. */
+ if (get_mempolicy(NULL, NULL, 0, 0, 0) < 0 && errno == ENOSYS) {
+ RTE_LOG(DEBUG, EAL, "NUMA is not supported.\n");
+ numa_available = false;
+ }
+
+ if (orig && numa_available) {
+ for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
+ if (internal_config.socket_mem[i])
+ maxnode = i + 1;
+ }
for (i = 0; i < hpi->num_pages[0]; i++) {
uint64_t hugepage_sz = hpi->hugepage_sz;
+ if (maxnode) {
+ unsigned int j;
+
+ for (j = 0; j < RTE_MAX_NUMA_NODES; j++)
+ if (essential_memory[j])
+ break;
+
+ if (j == RTE_MAX_NUMA_NODES) {
+ node_id = (node_id + 1) % RTE_MAX_NUMA_NODES;
+ while (!internal_config.socket_mem[node_id]) {
+ node_id++;
+ node_id %= RTE_MAX_NUMA_NODES;
+ }
+ } else {
+ node_id = j;
+ if (essential_memory[j] < hugepage_sz)
+ essential_memory[j] = 0;
+ else
+ essential_memory[j] -= hugepage_sz;
+ }
+
+ nodemask[node_id / ULONG_BITS] =
+ 1UL << (node_id % ULONG_BITS);
+
+ RTE_LOG(DEBUG, EAL,
+ "Setting policy MPOL_PREFERRED for socket %d\n",
+ node_id);
+ /*
+ * Due to old linux kernel bug (feature?) we have to
+ * increase maxnode by 1. It will be unconditionally
+ * decreased back to normal value inside the syscall
+ * handler.
+ */
+ if (set_mempolicy(MPOL_PREFERRED,
+ nodemask, maxnode + 1) < 0) {
+ RTE_LOG(ERR, EAL,
+ "Failed to set policy MPOL_PREFERRED: "
+ "%s\n", strerror(errno));
+ return i;
+ }
+
+ nodemask[node_id / ULONG_BITS] = 0UL;
+ }
+
if (orig) {
hugepg_tbl[i].file_id = i;
hugepg_tbl[i].size = hugepage_sz;
@@ -488,6 +562,9 @@ map_all_hugepages(struct hugepage_file *hugepg_tbl,
vma_len -= hugepage_sz;
}
+ if (maxnode && set_mempolicy(MPOL_DEFAULT, NULL, 0) < 0)
+ RTE_LOG(ERR, EAL, "Failed to set mempolicy MPOL_DEFAULT\n");
+
return i;
}
@@ -572,6 +649,9 @@ find_numasocket(struct hugepage_file *hugepg_tbl, struct hugepage_info *hpi)
if (hugepg_tbl[i].orig_va == va) {
hugepg_tbl[i].socket_id = socket_id;
hp_count++;
+ RTE_LOG(DEBUG, EAL,
+ "Hugepage %s is on socket %d\n",
+ hugepg_tbl[i].filepath, socket_id);
}
}
}
@@ -1010,6 +1090,11 @@ rte_eal_hugepage_init(void)
huge_register_sigbus();
+ /* make a copy of socket_mem, needed for balanced allocation. */
+ for (i = 0; i < RTE_MAX_NUMA_NODES; i++)
+ memory[i] = internal_config.socket_mem[i];
+
+
/* map all hugepages and sort them */
for (i = 0; i < (int)internal_config.num_hugepage_sizes; i ++){
unsigned pages_old, pages_new;
@@ -1027,7 +1112,8 @@ rte_eal_hugepage_init(void)
/* map all hugepages available */
pages_old = hpi->num_pages[0];
- pages_new = map_all_hugepages(&tmp_hp[hp_offset], hpi, 1);
+ pages_new = map_all_hugepages(&tmp_hp[hp_offset], hpi,
+ memory, 1);
if (pages_new < pages_old) {
RTE_LOG(DEBUG, EAL,
"%d not %d hugepages of size %u MB allocated\n",
@@ -1070,7 +1156,7 @@ rte_eal_hugepage_init(void)
sizeof(struct hugepage_file), cmp_physaddr);
/* remap all hugepages */
- if (map_all_hugepages(&tmp_hp[hp_offset], hpi, 0) !=
+ if (map_all_hugepages(&tmp_hp[hp_offset], hpi, NULL, 0) !=
hpi->num_pages[0]) {
RTE_LOG(ERR, EAL, "Failed to remap %u MB pages\n",
(unsigned)(hpi->hugepage_sz / 0x100000));
@@ -186,6 +186,7 @@ ifeq ($(CONFIG_RTE_BUILD_SHARED_LIB),n)
# The static libraries do not know their dependencies.
# So linking with static library requires explicit dependencies.
_LDLIBS-$(CONFIG_RTE_LIBRTE_EAL) += -lrt
+_LDLIBS-$(CONFIG_RTE_LIBRTE_EAL) += -lnuma
_LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED) += -lm
_LDLIBS-$(CONFIG_RTE_LIBRTE_SCHED) += -lrt
_LDLIBS-$(CONFIG_RTE_LIBRTE_METER) += -lm