From: David Lamparter Date: Wed, 19 Jun 2019 10:52:38 +0000 (+0200) Subject: lib/seqlock: avoid syscalls in no-waiter cases X-Git-Tag: base_7.2~75^2~3 X-Git-Url: https://git.puffer.fish/?a=commitdiff_plain;h=6046b690b53a5e1c14ccf261304f5aa1f53546ae;p=matthieu%2Ffrr.git lib/seqlock: avoid syscalls in no-waiter cases When we have no contention on the seqlock, we shouldn't incur the cost of syscalls. Signed-off-by: David Lamparter --- diff --git a/lib/seqlock.c b/lib/seqlock.c index 223d14952c..32dff5173d 100644 --- a/lib/seqlock.c +++ b/lib/seqlock.c @@ -103,16 +103,23 @@ void seqlock_wait(struct seqlock *sqlo, seqlock_val_t val) seqlock_assert_valid(val); wait_prep(sqlo); - while (1) { - cur = atomic_load_explicit(&sqlo->pos, memory_order_acquire); - if (!(cur & 1)) - break; - cal = cur - val - 1; + cur = atomic_load_explicit(&sqlo->pos, memory_order_relaxed); + + while (cur & SEQLOCK_HELD) { + cal = SEQLOCK_VAL(cur) - val - 1; assert(cal < 0x40000000 || cal > 0xc0000000); if (cal < 0x80000000) break; - wait_once(sqlo, cur); + if ((cur & SEQLOCK_WAITERS) + || atomic_compare_exchange_weak_explicit( + &sqlo->pos, &cur, cur | SEQLOCK_WAITERS, + memory_order_relaxed, memory_order_relaxed)) { + wait_once(sqlo, cur | SEQLOCK_WAITERS); + cur = atomic_load_explicit(&sqlo->pos, + memory_order_relaxed); + } + /* else: we failed to swap in cur because it just changed */ } wait_done(sqlo); } @@ -123,26 +130,32 @@ bool seqlock_check(struct seqlock *sqlo, seqlock_val_t val) seqlock_assert_valid(val); - cur = atomic_load_explicit(&sqlo->pos, memory_order_acquire); - if (!(cur & 1)) + cur = atomic_load_explicit(&sqlo->pos, memory_order_relaxed); + if (!(cur & SEQLOCK_HELD)) return 1; - cur -= val; + cur = SEQLOCK_VAL(cur) - val - 1; assert(cur < 0x40000000 || cur > 0xc0000000); return cur < 0x80000000; } void seqlock_acquire_val(struct seqlock *sqlo, seqlock_val_t val) { + seqlock_val_t prev; + seqlock_assert_valid(val); - atomic_store_explicit(&sqlo->pos, val, memory_order_release); - wait_poke(sqlo); + prev = atomic_exchange_explicit(&sqlo->pos, val, memory_order_relaxed); + if (prev & SEQLOCK_WAITERS) + wait_poke(sqlo); } void seqlock_release(struct seqlock *sqlo) { - atomic_store_explicit(&sqlo->pos, 0, memory_order_release); - wait_poke(sqlo); + seqlock_val_t prev; + + prev = atomic_exchange_explicit(&sqlo->pos, 0, memory_order_relaxed); + if (prev & SEQLOCK_WAITERS) + wait_poke(sqlo); } void seqlock_init(struct seqlock *sqlo) @@ -154,14 +167,23 @@ void seqlock_init(struct seqlock *sqlo) seqlock_val_t seqlock_cur(struct seqlock *sqlo) { - return atomic_load_explicit(&sqlo->pos, memory_order_acquire); + return SEQLOCK_VAL(atomic_load_explicit(&sqlo->pos, + memory_order_relaxed)); } seqlock_val_t seqlock_bump(struct seqlock *sqlo) { - seqlock_val_t val; + seqlock_val_t val, cur; + + cur = atomic_load_explicit(&sqlo->pos, memory_order_relaxed); + seqlock_assert_valid(cur); + + do { + val = SEQLOCK_VAL(cur) + SEQLOCK_INCR; + } while (!atomic_compare_exchange_weak_explicit(&sqlo->pos, &cur, val, + memory_order_relaxed, memory_order_relaxed)); - val = atomic_fetch_add_explicit(&sqlo->pos, 2, memory_order_release); - wait_poke(sqlo); + if (cur & SEQLOCK_WAITERS) + wait_poke(sqlo); return val; } diff --git a/lib/seqlock.h b/lib/seqlock.h index eef05a4307..0f5dc47cd7 100644 --- a/lib/seqlock.h +++ b/lib/seqlock.h @@ -54,12 +54,18 @@ */ /* use sequentially increasing "ticket numbers". lowest bit will always - * be 1 to have a 'cleared' indication (i.e., counts 1,3,5,7,etc. ) + * be 1 to have a 'cleared' indication (i.e., counts 1,5,9,13,etc. ) + * 2nd lowest bit is used to indicate we have waiters. */ typedef _Atomic uint32_t seqlock_ctr_t; typedef uint32_t seqlock_val_t; -#define seqlock_assert_valid(val) assert(val & 1) +#define seqlock_assert_valid(val) assert((val) & SEQLOCK_HELD) +#define SEQLOCK_HELD (1U << 0) +#define SEQLOCK_WAITERS (1U << 1) +#define SEQLOCK_VAL(n) ((n) & ~SEQLOCK_WAITERS) +#define SEQLOCK_STARTVAL 1U +#define SEQLOCK_INCR 4U struct seqlock { /* always used */ diff --git a/tests/lib/test_atomlist.c b/tests/lib/test_atomlist.c index 249fff8edb..238ee9539e 100644 --- a/tests/lib/test_atomlist.c +++ b/tests/lib/test_atomlist.c @@ -253,7 +253,7 @@ static void *thr1func(void *arg) struct testrun *tr; for (tr = runs; tr; tr = tr->next) { - sv = seqlock_bump(&p->sqlo); + sv = seqlock_bump(&p->sqlo) - SEQLOCK_INCR; seqlock_wait(&sqlo, sv); tr->func(offset); @@ -288,14 +288,14 @@ static void run_tr(struct testrun *tr) size_t c = 0, s = 0, n = 0; struct item *item, *prev, dummy; - printf("[%02u] %35s %s\n", seqlock_cur(&sqlo) >> 1, "", desc); + printf("[%02u] %35s %s\n", seqlock_cur(&sqlo) >> 2, "", desc); fflush(stdout); if (tr->prefill != NOCLEAR) clear_list(tr->prefill); monotime(&tv); - sv = seqlock_bump(&sqlo); + sv = seqlock_bump(&sqlo) - SEQLOCK_INCR; for (size_t i = 0; i < NTHREADS; i++) { seqlock_wait(&thr[i].sqlo, seqlock_cur(&sqlo)); s += thr[i].counter; @@ -325,7 +325,7 @@ static void run_tr(struct testrun *tr) assert(c == alist_count(&ahead)); } printf("\033[1A[%02u] %9"PRId64"us c=%5zu s=%5zu n=%5zu %s\n", - sv >> 1, delta, c, s, n, desc); + sv >> 2, delta, c, s, n, desc); } #ifdef BASIC_TESTS @@ -381,7 +381,7 @@ int main(int argc, char **argv) basic_tests(); seqlock_init(&sqlo); - seqlock_acquire_val(&sqlo, 1); + seqlock_acquire_val(&sqlo, SEQLOCK_STARTVAL); for (i = 0; i < NTHREADS; i++) { seqlock_init(&thr[i].sqlo); diff --git a/tests/lib/test_seqlock.c b/tests/lib/test_seqlock.c index 9cc6f80702..639c2bdc2b 100644 --- a/tests/lib/test_seqlock.c +++ b/tests/lib/test_seqlock.c @@ -66,20 +66,20 @@ static void *thr1func(void *arg) seqlock_wait(&sqlo, 1); writestr("thr1 @1\n"); - seqlock_wait(&sqlo, 3); - writestr("thr1 @3\n"); - seqlock_wait(&sqlo, 5); writestr("thr1 @5\n"); - seqlock_wait(&sqlo, 7); - writestr("thr1 @7\n"); - seqlock_wait(&sqlo, 9); writestr("thr1 @9\n"); - seqlock_wait(&sqlo, 11); - writestr("thr1 @11\n"); + seqlock_wait(&sqlo, 13); + writestr("thr1 @13\n"); + + seqlock_wait(&sqlo, 17); + writestr("thr1 @17\n"); + + seqlock_wait(&sqlo, 21); + writestr("thr1 @21\n"); return NULL; } @@ -95,11 +95,11 @@ int main(int argc, char **argv) assert(seqlock_cur(&sqlo) == 1); assert(seqlock_bump(&sqlo) == 1); - assert(seqlock_cur(&sqlo) == 3); - assert(seqlock_bump(&sqlo) == 3); + assert(seqlock_cur(&sqlo) == 5); assert(seqlock_bump(&sqlo) == 5); - assert(seqlock_bump(&sqlo) == 7); - assert(seqlock_cur(&sqlo) == 9); + assert(seqlock_bump(&sqlo) == 9); + assert(seqlock_bump(&sqlo) == 13); + assert(seqlock_cur(&sqlo) == 17); assert(seqlock_held(&sqlo)); seqlock_release(&sqlo); @@ -108,16 +108,16 @@ int main(int argc, char **argv) pthread_create(&thr1, NULL, thr1func, NULL); sleep(1); - writestr("main @3\n"); - seqlock_acquire_val(&sqlo, 3); + writestr("main @5\n"); + seqlock_acquire_val(&sqlo, 5); sleep(2); - writestr("main @5\n"); + writestr("main @9\n"); seqlock_bump(&sqlo); sleep(1); - writestr("main @9\n"); - seqlock_acquire_val(&sqlo, 9); + writestr("main @17\n"); + seqlock_acquire_val(&sqlo, 17); sleep(1); writestr("main @release\n");