lib: Memory spike reduction for sh cmds at scale

The output buffer vty->obuf is a linked list where
each element is of 4KB.
Currently, when a huge sh command  like <show ip route json>
is executed on a large scale, all the vty_outs are
processed and the entire data is accumulated.
After the entire vty execution, vtysh_flush proceeses
and puts this data in the socket (131KB at a time).

Problem here is the memory spike for such heavy duty
show commands.

The fix here is to chunkify the output on VTY shell by
flushing it intermediately for every 128 KB of output
accumulated and free the memory allocated for the buffer data.

This way, we achieve ~25-30% reduction in the memory spike.

Fixes: #16498
Note: This is a continuation of MR #16498

Signed-off-by: Srujana <skanchisamud@nvidia.com>
Signed-off-by: Rajasekar Raja <rajasekarr@nvidia.com>

diff --git a/lib/vty.c b/lib/vty.c
index 0a4a3d2b86a398f25b20b64d0c1267895afac25f..256a3bb3f538e3878dd67dcb0c5bf5944f8d7d64 100644 (file)
--- a/lib/vty.c
+++ b/lib/vty.c
@@ -345,8 +345,17 @@ int vty_out(struct vty *vty, const char *format, ...)
        case VTY_SHELL_SERV:
        case VTY_FILE:
        default:
+               vty->vty_buf_size_accumulated += strlen(filtered);
                /* print without crlf replacement */
                buffer_put(vty->obuf, (uint8_t *)filtered, strlen(filtered));
+               /* For every chunk of memory, we invoke vtysh_flush where we
+                * put the data of collective vty->obuf Linked List items on the
+                * socket and free the vty->obuf data.
+                */
+               if (vty->vty_buf_size_accumulated >= VTY_MAX_INTERMEDIATE_FLUSH) {
+                       vty->vty_buf_size_accumulated = 0;
+                       vtysh_flush(vty);
+               }
                break;
        }
 
@@ -2118,6 +2127,8 @@ static void vtysh_accept(struct event *thread)
        int client_len;
        struct sockaddr_un client;
        struct vty *vty;
+       int ret = 0;
+       uint32_t sndbufsize = VTY_SEND_BUF_MAX;
 
        vty_event_serv(VTYSH_SERV, vtyserv);
 
@@ -2141,6 +2152,20 @@ static void vtysh_accept(struct event *thread)
                close(sock);
                return;
        }
+
+       /*
+        * Increasing the SEND socket buffer size so that the socket can hold
+        * before sending it to VTY shell.
+        */
+       ret = setsockopt(sock, SOL_SOCKET, SO_SNDBUF, (char *)&sndbufsize,
+                        sizeof(sndbufsize));
+       if (ret < 0) {
+               flog_err(EC_LIB_SOCKET,
+                        "Cannot set socket %d send buffer size, %s", sock,
+                        safe_strerror(errno));
+               close(sock);
+               return;
+       }
        set_cloexec(sock);
 
 #ifdef VTYSH_DEBUG
@@ -2227,6 +2252,7 @@ static int vtysh_flush(struct vty *vty)
                vty_close(vty);
                return -1;
        case BUFFER_EMPTY:
+               vty->vty_buf_size_accumulated = 0;
                break;
        }
        return 0;

diff --git a/lib/vty.h b/lib/vty.h
index c336a816cc1fabd162cdcedd5795f3b11ca15a42..e511e8e79ae47108fc0810050a393eedaa144ff9 100644 (file)
--- a/lib/vty.h
+++ b/lib/vty.h
@@ -236,6 +236,7 @@ struct vty {
        uintptr_t mgmt_req_pending_data;
        bool mgmt_locked_candidate_ds;
        bool mgmt_locked_running_ds;
+       uint64_t vty_buf_size_accumulated;
 };
 
 static inline void vty_push_context(struct vty *vty, int node, uint64_t id)
@@ -338,6 +339,12 @@ struct vty_arg {
 /* Vty read buffer size. */
 #define VTY_READ_BUFSIZ 512
 
+/* Vty max send buffer size */
+#define VTY_SEND_BUF_MAX 16777216
+
+/* Vty flush intermediate size */
+#define VTY_MAX_INTERMEDIATE_FLUSH 131072
+
 /* Directory separator. */
 #ifndef DIRECTORY_SEP
 #define DIRECTORY_SEP '/'

diff --git a/vtysh/vtysh.c b/vtysh/vtysh.c
index c43e1909e3e197a5628841fdd9ae45d5482f425f..2d80feef6c970696c00b8b6443f604651674c645 100644 (file)
--- a/vtysh/vtysh.c
+++ b/vtysh/vtysh.c
@@ -4639,6 +4639,7 @@ static int vtysh_connect(struct vtysh_client *vclient)
        struct sockaddr_un addr;
        struct stat s_stat;
        const char *path;
+       uint32_t rcvbufsize = VTYSH_RCV_BUF_MAX;
 
        if (!vclient->path[0])
                snprintf(vclient->path, sizeof(vclient->path), "%s/%s.vty",
@@ -4688,6 +4689,22 @@ static int vtysh_connect(struct vtysh_client *vclient)
                close(sock);
                return -1;
        }
+
+       /*
+        * Increasing the RECEIVE socket buffer size so that the socket can hold
+        * after receving from other process.
+        */
+       ret = setsockopt(sock, SOL_SOCKET, SO_RCVBUF, (char *)&rcvbufsize,
+                        sizeof(rcvbufsize));
+       if (ret < 0) {
+#ifdef DEBUG
+               fprintf(stderr, "Cannot set socket %d rcv buffer size, %s\n",
+                       sock, safe_strerror(errno));
+#endif /* DEBUG */
+               close(sock);
+               return -1;
+       }
+
        vclient->fd = sock;
 
        return 0;

diff --git a/vtysh/vtysh.h b/vtysh/vtysh.h
index b1d57aa3c221027e1f22dad83e32cdb18120d9df..3c532b99a8cebe54fca5b8baab2a87c35a130639 100644 (file)
--- a/vtysh/vtysh.h
+++ b/vtysh/vtysh.h
@@ -36,6 +36,8 @@ extern struct event_loop *master;
 #define VTYSH_PIM6D     0x100000
 #define VTYSH_MGMTD 0x200000
 
+#define VTYSH_RCV_BUF_MAX 16777216
+
 #define VTYSH_WAS_ACTIVE (-2)
 
 /* commands in REALLYALL are crucial to correct vtysh operation */