zebra: misc fixes, perf improvements

* Coalesce multiple write() syscalls into one
* Write larger chunks
* Decrease default read limit to 1000
* Remove unnecessary operations from hot loop (zserv_write)
* Move cross-schedule out of obuf lock
* Use atomic ops to update atomic variable

Signed-off-by: Quentin Young <qlyoung@cumulusnetworks.com>

diff --git a/zebra/zebra_vty.c b/zebra/zebra_vty.c
index a094ca585e8d07dc3e6f326b967216f0ce76750b..a0eb66e847d1de8e235c71bd0231ae76e01e7114 100644 (file)
--- a/zebra/zebra_vty.c
+++ b/zebra/zebra_vty.c
@@ -3360,7 +3360,8 @@ DEFUN_HIDDEN (zebra_packet_process,
 {
        uint32_t packets = strtoul(argv[2]->arg, NULL, 10);
 
-       zebrad.packets_to_process = packets;
+       atomic_store_explicit(&zebrad.packets_to_process, packets,
+                             memory_order_relaxed);
 
        return CMD_SUCCESS;
 }
@@ -3373,7 +3374,9 @@ DEFUN_HIDDEN (no_zebra_packet_process,
              "Zapi Protocol\n"
              "Number of packets to process before relinquishing thread\n")
 {
-       zebrad.packets_to_process = ZEBRA_ZAPI_PACKETS_TO_PROCESS;
+       atomic_store_explicit(&zebrad.packets_to_process,
+                             ZEBRA_ZAPI_PACKETS_TO_PROCESS,
+                             memory_order_relaxed);
 
        return CMD_SUCCESS;
 }

diff --git a/zebra/zserv.c b/zebra/zserv.c
index 548e0b32c0cdddb33dfcf078721375c1ba1717d3..a264add420757a9480ed6fe883fc59b4ad72bd20 100644 (file)
--- a/zebra/zserv.c
+++ b/zebra/zserv.c
@@ -198,18 +198,19 @@ static int zserv_write(struct thread *thread)
        struct zserv *client = THREAD_ARG(thread);
        struct stream *msg;
        uint32_t wcmd;
-       int writerv;
        struct stream_fifo *cache;
 
        if (atomic_load_explicit(&client->dead, memory_order_seq_cst))
                return 0;
 
        /* If we have any data pending, try to flush it first */
-       switch (buffer_flush_available(client->wb, client->sock)) {
+       switch (buffer_flush_all(client->wb, client->sock)) {
        case BUFFER_ERROR:
                goto zwrite_fail;
        case BUFFER_PENDING:
-               client->last_write_time = monotime(NULL);
+               atomic_store_explicit(&client->last_write_time,
+                                     (uint32_t)monotime(NULL),
+                                     memory_order_relaxed);
                zserv_client_event(client, ZSERV_CLIENT_WRITE);
                return 0;
        case BUFFER_EMPTY:
@@ -226,32 +227,35 @@ static int zserv_write(struct thread *thread)
        }
        pthread_mutex_unlock(&client->obuf_mtx);
 
-       while (stream_fifo_head(cache)) {
-               msg = stream_fifo_pop(cache);
+       if (cache->tail) {
+               msg = cache->tail;
                stream_set_getp(msg, 0);
-
                wcmd = stream_getw_from(msg, 6);
-               writerv = buffer_write(client->wb, client->sock,
-                                      STREAM_DATA(msg), stream_get_endp(msg));
-
-               switch (writerv) {
-               case BUFFER_ERROR:
-                       stream_free(msg);
-                       stream_fifo_free(cache);
-                       goto zwrite_fail;
-               case BUFFER_PENDING:
-               case BUFFER_EMPTY:
-                       break;
-               }
+       }
 
+       while (stream_fifo_head(cache)) {
+               msg = stream_fifo_pop(cache);
+               buffer_put(client->wb, STREAM_DATA(msg), stream_get_endp(msg));
                stream_free(msg);
        }
 
-       if (!buffer_empty(client->wb))
-               zserv_client_event(client, ZSERV_CLIENT_WRITE);
-
        stream_fifo_free(cache);
 
+       /* If we have any data pending, try to flush it first */
+       switch (buffer_flush_all(client->wb, client->sock)) {
+       case BUFFER_ERROR:
+               goto zwrite_fail;
+       case BUFFER_PENDING:
+               atomic_store_explicit(&client->last_write_time,
+                                     (uint32_t)monotime(NULL),
+                                     memory_order_relaxed);
+               zserv_client_event(client, ZSERV_CLIENT_WRITE);
+               return 0;
+               break;
+       case BUFFER_EMPTY:
+               break;
+       }
+
        atomic_store_explicit(&client->last_write_cmd, wcmd,
                              memory_order_relaxed);
 
@@ -538,9 +542,11 @@ int zserv_send_message(struct zserv *client, struct stream *msg)
        pthread_mutex_lock(&client->obuf_mtx);
        {
                stream_fifo_push(client->obuf_fifo, msg);
-               zserv_client_event(client, ZSERV_CLIENT_WRITE);
        }
        pthread_mutex_unlock(&client->obuf_mtx);
+
+       zserv_client_event(client, ZSERV_CLIENT_WRITE);
+
        return 0;
 }
 

diff --git a/zebra/zserv.h b/zebra/zserv.h
index c8b65b6a4eb1e25827a03cb904267b5207a500f1..2a681552d16f583ff54e41be8cdf583babee6907 100644 (file)
--- a/zebra/zserv.h
+++ b/zebra/zserv.h
@@ -186,7 +186,7 @@ struct zebra_t {
        /* LSP work queue */
        struct work_queue *lsp_process_q;
 
-#define ZEBRA_ZAPI_PACKETS_TO_PROCESS 100000
+#define ZEBRA_ZAPI_PACKETS_TO_PROCESS 1000
        _Atomic uint32_t packets_to_process;
 };
 extern struct zebra_t zebrad;