Commit 1ef46e09 authored by Marek Vavrusa's avatar Marek Vavrusa

Resolved different cpu set APIs for NetBSD 5.0 and FreeBSD.

Each BSD flavor has slightly different way to handle CPU counting,
CPU set management and affinity setting.

refs #1882
parent ff5b8b0c
...@@ -144,7 +144,7 @@ AC_SEARCH_LIBS([crc32], [z]) ...@@ -144,7 +144,7 @@ AC_SEARCH_LIBS([crc32], [z])
# Checks for header files. # Checks for header files.
AC_HEADER_RESOLV AC_HEADER_RESOLV
AC_CHECK_HEADERS([arpa/inet.h fcntl.h inttypes.h limits.h malloc.h netdb.h netinet/in_systm.h netinet/in.h stdint.h stdlib.h string.h strings.h sys/socket.h sys/time.h cap-ng.h syslog.h unistd.h urcu.h ev.h]) AC_CHECK_HEADERS([arpa/inet.h fcntl.h inttypes.h limits.h malloc.h netdb.h netinet/in_systm.h netinet/in.h stdint.h stdlib.h string.h strings.h sys/socket.h sys/time.h cap-ng.h syslog.h unistd.h urcu.h ev.h pthread_np.h])
# Checks for typedefs, structures, and compiler characteristics. # Checks for typedefs, structures, and compiler characteristics.
AC_HEADER_STDBOOL AC_HEADER_STDBOOL
...@@ -166,6 +166,15 @@ AC_FUNC_FORK ...@@ -166,6 +166,15 @@ AC_FUNC_FORK
AC_FUNC_MMAP AC_FUNC_MMAP
AC_CHECK_FUNCS([gethostbyname gettimeofday clock_gettime memalign memmove memset munmap regcomp pselect select socket sqrt strcasecmp strchr strdup strerror strncasecmp strtol strtoul poll epoll_wait kqueue setgroups sendmmsg madvise pthread_setaffinity_np]) AC_CHECK_FUNCS([gethostbyname gettimeofday clock_gettime memalign memmove memset munmap regcomp pselect select socket sqrt strcasecmp strchr strdup strerror strncasecmp strtol strtoul poll epoll_wait kqueue setgroups sendmmsg madvise pthread_setaffinity_np])
# Check for cpu_set_t/cpuset_t compatibility
#AC_LANG([C])
AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <pthread.h>]], [[cpu_set_t set; CPU_ZERO(&set);]])],
[AC_DEFINE(HAVE_CPUSET_LINUX, 1, [Define if Linux-like cpu_set_t exists.])])
AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <pthread_np.h>]], [[cpuset_t set; CPU_ZERO(&set);]])],
[AC_DEFINE(HAVE_CPUSET_BSD, 1, [Define if FreeBSD-like cpuset_t exists.])])
AC_LINK_IFELSE([AC_LANG_PROGRAM([[#include <sched.h>]], [[cpuset_t* set = cpuset_create(); cpuset_destroy(set);]])],
[AC_DEFINE(HAVE_CPUSET_NETBSD, 1, [Define if cpuset_t and cpuset(3) exists.])])
AC_CONFIG_FILES([Makefile AC_CONFIG_FILES([Makefile
samples/Makefile samples/Makefile
src/Makefile]) src/Makefile])
......
...@@ -24,12 +24,20 @@ ...@@ -24,12 +24,20 @@
#ifdef HAVE_CAP_NG_H #ifdef HAVE_CAP_NG_H
#include <cap-ng.h> #include <cap-ng.h>
#endif /* HAVE_CAP_NG_H */ #endif /* HAVE_CAP_NG_H */
#ifdef HAVE_PTHREAD_NP_H
#include <pthread_np.h>
#endif /* HAVE_PTHREAD_NP_H */
#include "knot/common.h" #include "knot/common.h"
#include "knot/server/dthreads.h" #include "knot/server/dthreads.h"
#include "common/log.h" #include "common/log.h"
#include "knot/other/error.h" #include "knot/other/error.h"
/* BSD cpu set compatibility. */
#if defined(HAVE_CPUSET_BSD)
typedef cpuset_t cpu_set_t;
#endif
/*! \brief Lock thread state for R/W. */ /*! \brief Lock thread state for R/W. */
static inline void lock_thread_rw(dthread_t *thread) static inline void lock_thread_rw(dthread_t *thread)
{ {
...@@ -854,22 +862,44 @@ int dt_stop(dt_unit_t *unit) ...@@ -854,22 +862,44 @@ int dt_stop(dt_unit_t *unit)
// return KNOTD_EOK; // return KNOTD_EOK;
//} //}
int dt_setaffinity(dthread_t *thread, void *mask, size_t len)
int dt_setaffinity(dthread_t *thread, unsigned* cpu_id, size_t cpu_count)
{ {
if (thread == NULL || mask == NULL) { if (thread == NULL) {
return KNOTD_EINVAL; return KNOTD_EINVAL;
} }
#ifdef HAVE_PTHREAD_SETAFFINITY_NP #ifdef HAVE_PTHREAD_SETAFFINITY_NP
if (len != sizeof(cpu_set_t)) { int ret = -1;
return KNOTD_EINVAL;
/* Linux, FreeBSD interface. */
#if defined(HAVE_CPUSET_LINUX) || defined(HAVE_CPUSET_BSD)
cpu_set_t set;
CPU_ZERO(&set);
for (unsigned i = 0; i < cpu_count; ++i) {
CPU_SET(cpu_id[i], &set);
}
ret = pthread_setaffinity_np(thread->_thr, sizeof(cpu_set_t), &set);
/* NetBSD interface. */
#elif defined(HAVE_CPUSET_NETBSD)
cpuset_t *set = cpuset_create();
if (set == NULL) {
return KNOTD_ENOMEM;
}
cpuset_zero(set);
for (unsigned i = 0; i < cpu_count; ++i) {
cpuset_set(cpu_id[i], &set);
} }
pthread_t tid = pthread_self(); ret = pthread_setaffinity_np(thread->_thr, cpuset_size(set), set);
int ret = pthread_setaffinity_np(tid, len, (cpu_set_t*)mask); cpuset_destroy(set);
#endif /* interface */
if (ret < 0) { if (ret < 0) {
return KNOTD_ERROR; return KNOTD_ERROR;
} }
#else
#else /* HAVE_PTHREAD_SETAFFINITY_NP */
return KNOTD_ENOTSUP; return KNOTD_ENOTSUP;
#endif #endif
...@@ -988,8 +1018,17 @@ int dt_compact(dt_unit_t *unit) ...@@ -988,8 +1018,17 @@ int dt_compact(dt_unit_t *unit)
int dt_online_cpus() int dt_online_cpus()
{ {
int ret = -1; int ret = -1;
/* Linux, Solaris, OS X 10.4+ */
#ifdef _SC_NPROCESSORS_ONLN #ifdef _SC_NPROCESSORS_ONLN
ret = (int) sysconf(_SC_NPROCESSORS_ONLN); ret = (int) sysconf(_SC_NPROCESSORS_ONLN);
#else
/* FreeBSD, NetBSD, OpenBSD, OS X < 10.4 */
#if HAVE_SYSCTLBYNAME
size_t rlen = sizeof(int);
if (sysctlbyname("hw.ncpu", &ret, &rlen, NULL, 0) < 0) {
ret = -1;
}
#endif
#endif #endif
return ret; return ret;
} }
......
...@@ -254,12 +254,13 @@ int dt_stop(dt_unit_t *unit); ...@@ -254,12 +254,13 @@ int dt_stop(dt_unit_t *unit);
* \brief Set thread affinity to masked CPU's. * \brief Set thread affinity to masked CPU's.
* *
* \param thread Target thread instance. * \param thread Target thread instance.
* \param mask CPU mask (should be pointer to cpu_set_t). * \param cpu_id Array of CPU IDs to set affinity to.
* \param cpu_count Number of CPUs in the array, set to 0 for no CPU.
* *
* \retval KNOTD_EOK on success. * \retval KNOTD_EOK on success.
* \retval KNOTD_EINVAL on invalid parameters. * \retval KNOTD_EINVAL on invalid parameters.
*/ */
int dt_setaffinity(dthread_t *thread, void *mask, size_t len); int dt_setaffinity(dthread_t *thread, unsigned* cpu_id, size_t cpu_count);
/*! /*!
* \brief Set thread to execute another runnable. * \brief Set thread to execute another runnable.
......
...@@ -204,17 +204,14 @@ static inline int udp_master_recvfrom(dthread_t *thread, stat_t *thread_stat) ...@@ -204,17 +204,14 @@ static inline int udp_master_recvfrom(dthread_t *thread, stat_t *thread_stat)
/* Set CPU affinity to improve load distribution on multicore systems. /* Set CPU affinity to improve load distribution on multicore systems.
* Partial overlapping mask to be nice to scheduler. * Partial overlapping mask to be nice to scheduler.
*/ */
#ifdef HAVE_PTHREAD_SETAFFINITY_NP
int cpcount = dt_online_cpus(); int cpcount = dt_online_cpus();
if (cpcount > 0) { if (cpcount > 0) {
unsigned tid = dt_get_id(thread); unsigned cpu[2];
cpu_set_t cpus; cpu[0] = dt_get_id(thread);
CPU_ZERO(&cpus); cpu[1] = (cpu[0] + 1) % cpcount;
CPU_SET(tid % cpcount, &cpus); cpu[0] = cpu[0] % cpcount;
CPU_SET((tid + 1) % cpcount, &cpus); dt_setaffinity(thread, cpu, 2);
dt_setaffinity(thread, &cpus, sizeof(cpu_set_t));
} }
#endif
knot_nameserver_t *ns = h->server->nameserver; knot_nameserver_t *ns = h->server->nameserver;
...@@ -407,17 +404,14 @@ static inline int udp_master_recvmmsg(dthread_t *thread, stat_t *thread_stat) ...@@ -407,17 +404,14 @@ static inline int udp_master_recvmmsg(dthread_t *thread, stat_t *thread_stat)
/* Set CPU affinity to improve load distribution on multicore systems. /* Set CPU affinity to improve load distribution on multicore systems.
* Partial overlapping mask to be nice to scheduler. * Partial overlapping mask to be nice to scheduler.
*/ */
#ifdef HAVE_PTHREAD_SETAFFINITY_NP
int cpcount = dt_online_cpus(); int cpcount = dt_online_cpus();
if (cpcount > 0) { if (cpcount > 0) {
unsigned tid = dt_get_id(thread); unsigned cpu[2];
cpu_set_t cpus; cpu[0] = dt_get_id(thread);
CPU_ZERO(&cpus); cpu[1] = (cpu[0] + 1) % cpcount;
CPU_SET(tid % cpcount, &cpus); cpu[0] = cpu[0] % cpcount;
CPU_SET((tid + 1) % cpcount, &cpus); dt_setaffinity(thread, cpu, 2);
dt_setaffinity(thread, &cpus, sizeof(cpu_set_t));
} }
#endif
/* Loop until all data is read. */ /* Loop until all data is read. */
ssize_t n = 0; ssize_t n = 0;
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment