lib/seqlock: avoid syscalls in no-waiter cases

When we have no contention on the seqlock, we shouldn't incur the cost
of syscalls.

Signed-off-by: David Lamparter <equinox@opensourcerouting.org>

diff --git a/lib/seqlock.c b/lib/seqlock.c
index 223d14952ceb5b4a164ae428377a3ebe8bee6e73..32dff5173d8d755c2d23aab69a1497f63e44b7e6 100644 (file)
--- a/lib/seqlock.c
+++ b/lib/seqlock.c
@@ -103,16 +103,23 @@ void seqlock_wait(struct seqlock *sqlo, seqlock_val_t val)
        seqlock_assert_valid(val);
 
        wait_prep(sqlo);
-       while (1) {
-               cur = atomic_load_explicit(&sqlo->pos, memory_order_acquire);
-               if (!(cur & 1))
-                       break;
-               cal = cur - val - 1;
+       cur = atomic_load_explicit(&sqlo->pos, memory_order_relaxed);
+
+       while (cur & SEQLOCK_HELD) {
+               cal = SEQLOCK_VAL(cur) - val - 1;
                assert(cal < 0x40000000 || cal > 0xc0000000);
                if (cal < 0x80000000)
                        break;
 
-               wait_once(sqlo, cur);
+               if ((cur & SEQLOCK_WAITERS)
+                   || atomic_compare_exchange_weak_explicit(
+                               &sqlo->pos, &cur, cur | SEQLOCK_WAITERS,
+                               memory_order_relaxed, memory_order_relaxed)) {
+                       wait_once(sqlo, cur | SEQLOCK_WAITERS);
+                       cur = atomic_load_explicit(&sqlo->pos,
+                               memory_order_relaxed);
+               }
+               /* else: we failed to swap in cur because it just changed */
        }
        wait_done(sqlo);
 }
@@ -123,26 +130,32 @@ bool seqlock_check(struct seqlock *sqlo, seqlock_val_t val)
 
        seqlock_assert_valid(val);
 
-       cur = atomic_load_explicit(&sqlo->pos, memory_order_acquire);
-       if (!(cur & 1))
+       cur = atomic_load_explicit(&sqlo->pos, memory_order_relaxed);
+       if (!(cur & SEQLOCK_HELD))
                return 1;
-       cur -= val;
+       cur = SEQLOCK_VAL(cur) - val - 1;
        assert(cur < 0x40000000 || cur > 0xc0000000);
        return cur < 0x80000000;
 }
 
 void seqlock_acquire_val(struct seqlock *sqlo, seqlock_val_t val)
 {
+       seqlock_val_t prev;
+
        seqlock_assert_valid(val);
 
-       atomic_store_explicit(&sqlo->pos, val, memory_order_release);
-       wait_poke(sqlo);
+       prev = atomic_exchange_explicit(&sqlo->pos, val, memory_order_relaxed);
+       if (prev & SEQLOCK_WAITERS)
+               wait_poke(sqlo);
 }
 
 void seqlock_release(struct seqlock *sqlo)
 {
-       atomic_store_explicit(&sqlo->pos, 0, memory_order_release);
-       wait_poke(sqlo);
+       seqlock_val_t prev;
+
+       prev = atomic_exchange_explicit(&sqlo->pos, 0, memory_order_relaxed);
+       if (prev & SEQLOCK_WAITERS)
+               wait_poke(sqlo);
 }
 
 void seqlock_init(struct seqlock *sqlo)
@@ -154,14 +167,23 @@ void seqlock_init(struct seqlock *sqlo)
 
 seqlock_val_t seqlock_cur(struct seqlock *sqlo)
 {
-       return atomic_load_explicit(&sqlo->pos, memory_order_acquire);
+       return SEQLOCK_VAL(atomic_load_explicit(&sqlo->pos,
+                                               memory_order_relaxed));
 }
 
 seqlock_val_t seqlock_bump(struct seqlock *sqlo)
 {
-       seqlock_val_t val;
+       seqlock_val_t val, cur;
+
+       cur = atomic_load_explicit(&sqlo->pos, memory_order_relaxed);
+       seqlock_assert_valid(cur);
+
+       do {
+               val = SEQLOCK_VAL(cur) + SEQLOCK_INCR;
+       } while (!atomic_compare_exchange_weak_explicit(&sqlo->pos, &cur, val,
+                       memory_order_relaxed, memory_order_relaxed));
 
-       val = atomic_fetch_add_explicit(&sqlo->pos, 2, memory_order_release);
-       wait_poke(sqlo);
+       if (cur & SEQLOCK_WAITERS)
+               wait_poke(sqlo);
        return val;
 }

diff --git a/lib/seqlock.h b/lib/seqlock.h
index eef05a4307ec5168703e3a5df44ad6c8586ab3bc..0f5dc47cd70fdc2c503978a1d329ef1f76582e62 100644 (file)
--- a/lib/seqlock.h
+++ b/lib/seqlock.h
@@ -54,12 +54,18 @@
  */
 
 /* use sequentially increasing "ticket numbers".  lowest bit will always
- * be 1 to have a 'cleared' indication (i.e., counts 1,3,5,7,etc. )
+ * be 1 to have a 'cleared' indication (i.e., counts 1,5,9,13,etc. )
+ * 2nd lowest bit is used to indicate we have waiters.
  */
 typedef _Atomic uint32_t       seqlock_ctr_t;
 typedef uint32_t               seqlock_val_t;
-#define seqlock_assert_valid(val) assert(val & 1)
+#define seqlock_assert_valid(val) assert((val) & SEQLOCK_HELD)
 
+#define SEQLOCK_HELD           (1U << 0)
+#define SEQLOCK_WAITERS                (1U << 1)
+#define SEQLOCK_VAL(n)         ((n) & ~SEQLOCK_WAITERS)
+#define SEQLOCK_STARTVAL       1U
+#define SEQLOCK_INCR           4U
 
 struct seqlock {
 /* always used */

diff --git a/tests/lib/test_atomlist.c b/tests/lib/test_atomlist.c
index 249fff8edb34e2659283e80954e8036886b66133..238ee9539e0add36592e4c0e7cf049fe250e5130 100644 (file)
--- a/tests/lib/test_atomlist.c
+++ b/tests/lib/test_atomlist.c
@@ -253,7 +253,7 @@ static void *thr1func(void *arg)
        struct testrun *tr;
 
        for (tr = runs; tr; tr = tr->next) {
-               sv = seqlock_bump(&p->sqlo);
+               sv = seqlock_bump(&p->sqlo) - SEQLOCK_INCR;
                seqlock_wait(&sqlo, sv);
 
                tr->func(offset);
@@ -288,14 +288,14 @@ static void run_tr(struct testrun *tr)
        size_t c = 0, s = 0, n = 0;
        struct item *item, *prev, dummy;
 
-       printf("[%02u] %35s %s\n", seqlock_cur(&sqlo) >> 1, "", desc);
+       printf("[%02u] %35s %s\n", seqlock_cur(&sqlo) >> 2, "", desc);
        fflush(stdout);
 
        if (tr->prefill != NOCLEAR)
                clear_list(tr->prefill);
 
        monotime(&tv);
-       sv = seqlock_bump(&sqlo);
+       sv = seqlock_bump(&sqlo) - SEQLOCK_INCR;
        for (size_t i = 0; i < NTHREADS; i++) {
                seqlock_wait(&thr[i].sqlo, seqlock_cur(&sqlo));
                s += thr[i].counter;
@@ -325,7 +325,7 @@ static void run_tr(struct testrun *tr)
                assert(c == alist_count(&ahead));
        }
        printf("\033[1A[%02u] %9"PRId64"us c=%5zu s=%5zu n=%5zu %s\n",
-               sv >> 1, delta, c, s, n, desc);
+               sv >> 2, delta, c, s, n, desc);
 }
 
 #ifdef BASIC_TESTS
@@ -381,7 +381,7 @@ int main(int argc, char **argv)
        basic_tests();
 
        seqlock_init(&sqlo);
-       seqlock_acquire_val(&sqlo, 1);
+       seqlock_acquire_val(&sqlo, SEQLOCK_STARTVAL);
 
        for (i = 0; i < NTHREADS; i++) {
                seqlock_init(&thr[i].sqlo);

diff --git a/tests/lib/test_seqlock.c b/tests/lib/test_seqlock.c
index 9cc6f80702c2b68f3052c5626aa73dc525742dcf..639c2bdc2bc1c25628a634067eb93e121bb66dfb 100644 (file)
--- a/tests/lib/test_seqlock.c
+++ b/tests/lib/test_seqlock.c
@@ -66,20 +66,20 @@ static void *thr1func(void *arg)
        seqlock_wait(&sqlo, 1);
        writestr("thr1 @1\n");
 
-       seqlock_wait(&sqlo, 3);
-       writestr("thr1 @3\n");
-
        seqlock_wait(&sqlo, 5);
        writestr("thr1 @5\n");
 
-       seqlock_wait(&sqlo, 7);
-       writestr("thr1 @7\n");
-
        seqlock_wait(&sqlo, 9);
        writestr("thr1 @9\n");
 
-       seqlock_wait(&sqlo, 11);
-       writestr("thr1 @11\n");
+       seqlock_wait(&sqlo, 13);
+       writestr("thr1 @13\n");
+
+       seqlock_wait(&sqlo, 17);
+       writestr("thr1 @17\n");
+
+       seqlock_wait(&sqlo, 21);
+       writestr("thr1 @21\n");
        return NULL;
 }
 
@@ -95,11 +95,11 @@ int main(int argc, char **argv)
 
        assert(seqlock_cur(&sqlo) == 1);
        assert(seqlock_bump(&sqlo) == 1);
-       assert(seqlock_cur(&sqlo) == 3);
-       assert(seqlock_bump(&sqlo) == 3);
+       assert(seqlock_cur(&sqlo) == 5);
        assert(seqlock_bump(&sqlo) == 5);
-       assert(seqlock_bump(&sqlo) == 7);
-       assert(seqlock_cur(&sqlo) == 9);
+       assert(seqlock_bump(&sqlo) == 9);
+       assert(seqlock_bump(&sqlo) == 13);
+       assert(seqlock_cur(&sqlo) == 17);
 
        assert(seqlock_held(&sqlo));
        seqlock_release(&sqlo);
@@ -108,16 +108,16 @@ int main(int argc, char **argv)
        pthread_create(&thr1, NULL, thr1func, NULL);
        sleep(1);
 
-       writestr("main @3\n");
-       seqlock_acquire_val(&sqlo, 3);
+       writestr("main @5\n");
+       seqlock_acquire_val(&sqlo, 5);
        sleep(2);
 
-       writestr("main @5\n");
+       writestr("main @9\n");
        seqlock_bump(&sqlo);
        sleep(1);
 
-       writestr("main @9\n");
-       seqlock_acquire_val(&sqlo, 9);
+       writestr("main @17\n");
+       seqlock_acquire_val(&sqlo, 17);
        sleep(1);
 
        writestr("main @release\n");