From fb0aa88623f72b7e4d0f35e8df3f96aa090fc5c8 Mon Sep 17 00:00:00 2001 From: Avneesh Sachdev Date: Mon, 4 Apr 2016 10:54:58 -0700 Subject: [PATCH] zebra: optionally use protobuf with FPM Change zebra so that it can optionally use protobuf serialization when communicating with a Forwarding Plane Manager component. * zebra/main.c Add the --fpm-format/-F command line option. This allows the user to control the format (protbuf|netlink) that is used to communicate with the FPM. * zebra/zebra_fpm.c - zebra_init_msg_format(), This new function is invoked on process startup to determine the FPM format that should be used. - zfpm_init() Change to accept any 'FPM message format' specified by the user (via the new command line flag). - zebra_encode_route() Tweak to use the selected FPM format. * zebra_fpm_protobuf.c New code to build protobuf messages to be sent to the FPM. * zebra/Makefile.am - Include common.am - Build new file zebra_fpm_protobuf.c when protobuf is available. - Link with the fpm_pb library. Signed-off-by: Avneesh Sachdev --- zebra/Makefile.am | 11 +- zebra/main.c | 14 +- zebra/zebra_fpm.c | 139 ++++++++++++++--- zebra/zebra_fpm.h | 3 +- zebra/zebra_fpm_private.h | 5 + zebra/zebra_fpm_protobuf.c | 311 +++++++++++++++++++++++++++++++++++++ 6 files changed, 455 insertions(+), 28 deletions(-) create mode 100644 zebra/zebra_fpm_protobuf.c diff --git a/zebra/Makefile.am b/zebra/Makefile.am index 851e597796..26c283b9c1 100644 --- a/zebra/Makefile.am +++ b/zebra/Makefile.am @@ -1,3 +1,5 @@ +include ../common.am + ## Process this file with automake to produce Makefile.in. AM_CPPFLAGS = -I.. -I$(top_srcdir) -I$(top_srcdir)/lib -I$(top_builddir)/lib @@ -21,6 +23,10 @@ if HAVE_NETLINK othersrc = zebra_fpm_netlink.c endif +if HAVE_PROTOBUF +protobuf_srcs = zebra_fpm_protobuf.c +endif + AM_CFLAGS = $(WERROR) sbin_PROGRAMS = zebra @@ -33,7 +39,8 @@ zebra_SOURCES = \ redistribute.c debug.c rtadv.c zebra_snmp.c zebra_vty.c \ irdp_main.c irdp_interface.c irdp_packet.c router-id.c zebra_fpm.c \ $(othersrc) zebra_ptm.c zebra_rnh.c zebra_ptm_redistribute.c \ - zebra_ns.c zebra_vrf.c zebra_static.c zebra_mpls.c zebra_mpls_vty.c + zebra_ns.c zebra_vrf.c zebra_static.c zebra_mpls.c zebra_mpls_vty.c \ + $(protobuf_srcs) testzebra_SOURCES = test_main.c zebra_rib.c interface.c connected.c debug.c \ zebra_vty.c zebra_ptm.c zebra_routemap.c zebra_ns.c zebra_vrf.c \ @@ -49,7 +56,7 @@ noinst_HEADERS = \ zebra_ptm_redistribute.h zebra_ptm.h zebra_routemap.h \ zebra_ns.h zebra_vrf.h ioctl_solaris.h zebra_static.h zebra_mpls.h -zebra_LDADD = $(otherobj) ../lib/libzebra.la $(LIBCAP) +zebra_LDADD = $(otherobj) ../lib/libzebra.la $(LIBCAP) $(Q_FPM_PB_CLIENT_LDOPTS) testzebra_LDADD = ../lib/libzebra.la $(LIBCAP) diff --git a/zebra/main.c b/zebra/main.c index faa6cdb317..da7e6b6fb8 100644 --- a/zebra/main.c +++ b/zebra/main.c @@ -83,6 +83,7 @@ struct option longopts[] = { "daemon", no_argument, NULL, 'd'}, { "allow_delete", no_argument, NULL, 'a'}, { "keep_kernel", no_argument, NULL, 'k'}, + { "fpm_format", required_argument, NULL, 'F'}, { "config_file", required_argument, NULL, 'f'}, { "pid_file", required_argument, NULL, 'i'}, { "socket", required_argument, NULL, 'z'}, @@ -143,6 +144,7 @@ usage (char *progname, int status) "-d, --daemon Runs in daemon mode\n"\ "-a, --allow_delete Allow other processes to delete Quagga Routes\n" \ "-f, --config_file Set configuration file name\n"\ + "-F, --fpm_format Set fpm format to 'netlink' or 'protobuf'\n"\ "-i, --pid_file Set process identifier file name\n"\ "-z, --socket Set path of zebra socket\n"\ "-k, --keep_kernel Don't delete old routes which installed by "\ @@ -238,6 +240,7 @@ main (int argc, char **argv) char *progname; struct thread thread; char *zserv_path = NULL; + char *fpm_format = NULL; /* Set umask before anything for security */ umask (0027); @@ -257,9 +260,9 @@ main (int argc, char **argv) int opt; #ifdef HAVE_NETLINK - opt = getopt_long (argc, argv, "bdakf:i:z:hA:P:ru:g:vs:C", longopts, 0); + opt = getopt_long (argc, argv, "bdakf:F:i:z:hA:P:ru:g:vs:C", longopts, 0); #else - opt = getopt_long (argc, argv, "bdakf:i:z:hA:P:ru:g:vC", longopts, 0); + opt = getopt_long (argc, argv, "bdakf:F:i:z:hA:P:ru:g:vC", longopts, 0); #endif /* HAVE_NETLINK */ if (opt == EOF) @@ -286,6 +289,9 @@ main (int argc, char **argv) case 'f': config_file = optarg; break; + case 'F': + fpm_format = optarg; + break; case 'A': vty_addr = optarg; break; @@ -377,9 +383,9 @@ main (int argc, char **argv) #endif /* HAVE_SNMP */ #ifdef HAVE_FPM - zfpm_init (zebrad.master, 1, 0); + zfpm_init (zebrad.master, 1, 0, fpm_format); #else - zfpm_init (zebrad.master, 0, 0); + zfpm_init (zebrad.master, 0, 0, fpm_format); #endif /* Process the configuration file. Among other configuration diff --git a/zebra/zebra_fpm.c b/zebra/zebra_fpm.c index 34f0068514..5a68bcbfac 100644 --- a/zebra/zebra_fpm.c +++ b/zebra/zebra_fpm.c @@ -141,6 +141,15 @@ typedef enum { } zfpm_state_t; +/* + * Message format to be used to communicate with the FPM. + */ +typedef enum +{ + ZFPM_MSG_FORMAT_NONE, + ZFPM_MSG_FORMAT_NETLINK, + ZFPM_MSG_FORMAT_PROTOBUF, +} zfpm_msg_format_e; /* * Globals. */ @@ -152,6 +161,11 @@ typedef struct zfpm_glob_t_ */ int enabled; + /* + * Message format to be used to communicate with the fpm. + */ + zfpm_msg_format_e message_format; + struct thread_master *master; zfpm_state_t state; @@ -866,19 +880,40 @@ zfpm_writes_pending (void) */ static inline int zfpm_encode_route (rib_dest_t *dest, struct rib *rib, char *in_buf, - size_t in_buf_len) + size_t in_buf_len, fpm_msg_type_e *msg_type) { -#ifndef HAVE_NETLINK - return 0; -#else - + size_t len; int cmd; + len = 0; - cmd = rib ? RTM_NEWROUTE : RTM_DELROUTE; + *msg_type = FPM_MSG_TYPE_NONE; - return zfpm_netlink_encode_route (cmd, dest, rib, in_buf, in_buf_len); + switch (zfpm_g->message_format) { + case ZFPM_MSG_FORMAT_PROTOBUF: +#ifdef HAVE_PROTOBUF + len = zfpm_protobuf_encode_route (dest, rib, (uint8_t *) in_buf, + in_buf_len); + *msg_type = FPM_MSG_TYPE_PROTOBUF; +#endif + break; + + case ZFPM_MSG_FORMAT_NETLINK: +#ifdef HAVE_NETLINK + *msg_type = FPM_MSG_TYPE_NETLINK; + cmd = rib ? RTM_NEWROUTE : RTM_DELROUTE; + len = zfpm_netlink_encode_route (cmd, dest, rib, in_buf, in_buf_len); + assert(fpm_msg_align(len) == len); + *msg_type = FPM_MSG_TYPE_NETLINK; #endif /* HAVE_NETLINK */ + break; + + default: + break; + } + + return len; + } /* @@ -886,7 +921,7 @@ zfpm_encode_route (rib_dest_t *dest, struct rib *rib, char *in_buf, * * Returns the rib that is to be sent to the FPM for a given dest. */ -static struct rib * +struct rib * zfpm_route_for_update (rib_dest_t *dest) { struct rib *rib; @@ -922,6 +957,7 @@ zfpm_build_updates (void) fpm_msg_hdr_t *hdr; struct rib *rib; int is_add, write_msg; + fpm_msg_type_e msg_type; s = zfpm_g->obuf; @@ -946,7 +982,6 @@ zfpm_build_updates (void) hdr = (fpm_msg_hdr_t *) buf; hdr->version = FPM_PROTO_VERSION; - hdr->msg_type = FPM_MSG_TYPE_NETLINK; data = fpm_msg_data (hdr); @@ -966,11 +1001,13 @@ zfpm_build_updates (void) } if (write_msg) { - data_len = zfpm_encode_route (dest, rib, (char *) data, buf_end - data); + data_len = zfpm_encode_route (dest, rib, (char *) data, buf_end - data, + &msg_type); assert (data_len); if (data_len) { + hdr->msg_type = msg_type; msg_len = fpm_data_len_to_msg_len (data_len); hdr->msg_len = htons (msg_len); stream_forward_endp (s, msg_len); @@ -1572,6 +1609,64 @@ DEFUN ( no_fpm_remote_ip, } +/* + * zfpm_init_message_format + */ +static inline void +zfpm_init_message_format (const char *format) +{ + int have_netlink, have_protobuf; + + have_netlink = have_protobuf = 0; + +#ifdef HAVE_NETLINK + have_netlink = 1; +#endif + +#ifdef HAVE_PROTOBUF + have_protobuf = 1; +#endif + + zfpm_g->message_format = ZFPM_MSG_FORMAT_NONE; + + if (!format) + { + if (have_netlink) + { + zfpm_g->message_format = ZFPM_MSG_FORMAT_NETLINK; + } + else if (have_protobuf) + { + zfpm_g->message_format = ZFPM_MSG_FORMAT_PROTOBUF; + } + return; + } + + if (!strcmp ("netlink", format)) + { + if (!have_netlink) + { + zlog_err ("FPM netlink message format is not available"); + return; + } + zfpm_g->message_format = ZFPM_MSG_FORMAT_NETLINK; + return; + } + + if (!strcmp ("protobuf", format)) + { + if (!have_protobuf) + { + zlog_err ("FPM protobuf message format is not available"); + return; + } + zfpm_g->message_format = ZFPM_MSG_FORMAT_PROTOBUF; + return; + } + + zlog_warn ("Unknown fpm format '%s'", format); +} + /** * fpm_remote_srv_write * @@ -1601,11 +1696,13 @@ int fpm_remote_srv_write (struct vty *vty ) * * @param[in] port port at which FPM is running. * @param[in] enable TRUE if the zebra FPM module should be enabled + * @param[in] format to use to talk to the FPM. Can be 'netink' or 'protobuf'. * * Returns TRUE on success. */ int -zfpm_init (struct thread_master *master, int enable, uint16_t port) +zfpm_init (struct thread_master *master, int enable, uint16_t port, + const char *format) { static int initialized = 0; @@ -1621,16 +1718,6 @@ zfpm_init (struct thread_master *master, int enable, uint16_t port) zfpm_g->sock = -1; zfpm_g->state = ZFPM_STATE_IDLE; - /* - * Netlink must currently be available for the Zebra-FPM interface - * to be enabled. - */ -#ifndef HAVE_NETLINK - enable = 0; -#endif - - zfpm_g->enabled = enable; - zfpm_stats_init (&zfpm_g->stats); zfpm_stats_init (&zfpm_g->last_ivl_stats); zfpm_stats_init (&zfpm_g->cumulative_stats); @@ -1640,6 +1727,16 @@ zfpm_init (struct thread_master *master, int enable, uint16_t port) install_element (CONFIG_NODE, &fpm_remote_ip_cmd); install_element (CONFIG_NODE, &no_fpm_remote_ip_cmd); + zfpm_init_message_format(format); + + /* + * Disable FPM interface if no suitable format is available. + */ + if (zfpm_g->message_format == ZFPM_MSG_FORMAT_NONE) + enable = 0; + + zfpm_g->enabled = enable; + if (!enable) { return 1; } diff --git a/zebra/zebra_fpm.h b/zebra/zebra_fpm.h index 7fa71e40bc..fdb069965b 100644 --- a/zebra/zebra_fpm.h +++ b/zebra/zebra_fpm.h @@ -28,7 +28,8 @@ /* * Externs. */ -extern int zfpm_init (struct thread_master *master, int enable, uint16_t port); +extern int zfpm_init (struct thread_master *master, int enable, uint16_t port, + const char *message_format); extern void zfpm_trigger_update (struct route_node *rn, const char *reason); extern int fpm_remote_srv_write (struct vty *vty); diff --git a/zebra/zebra_fpm_private.h b/zebra/zebra_fpm_private.h index 809a70a445..1c4fd4c22f 100644 --- a/zebra/zebra_fpm_private.h +++ b/zebra/zebra_fpm_private.h @@ -53,4 +53,9 @@ extern int zfpm_netlink_encode_route (int cmd, rib_dest_t *dest, struct rib *rib, char *in_buf, size_t in_buf_len); +extern int +zfpm_protobuf_encode_route (rib_dest_t *dest, struct rib *rib, + uint8_t *in_buf, size_t in_buf_len); + +extern struct rib *zfpm_route_for_update (rib_dest_t *dest); #endif /* _ZEBRA_FPM_PRIVATE_H */ diff --git a/zebra/zebra_fpm_protobuf.c b/zebra/zebra_fpm_protobuf.c new file mode 100644 index 0000000000..beef310b17 --- /dev/null +++ b/zebra/zebra_fpm_protobuf.c @@ -0,0 +1,311 @@ +/* + * zebra_fpm_protobuf.c + * + * @copyright Copyright (C) 2016 Sproute Networks, Inc. + * + * @author Avneesh Sachdev + * + * This file is part of Quagga. + * + * Quagga is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2, or (at your option) any + * later version. + * + * Quagga is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Quagga; see the file COPYING. If not, write to the Free + * Software Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA + * 02111-1307, USA. + */ +#include + +#include "log.h" +#include "rib.h" + +#include "qpb/qpb.pb-c.h" +#include "qpb/qpb.h" +#include "qpb/qpb_allocator.h" +#include "qpb/linear_allocator.h" +#include "fpm/fpm_pb.h" + +#include "zebra_fpm_private.h" + +/* + * create_delete_route_message + */ +static Fpm__DeleteRoute * +create_delete_route_message (qpb_allocator_t *allocator, rib_dest_t *dest, + struct rib *rib) +{ + Fpm__DeleteRoute *msg; + + msg = QPB_ALLOC(allocator, typeof(*msg)); + if (!msg) { + assert(0); + return NULL; + } + + fpm__delete_route__init(msg); + msg->vrf_id = rib_dest_vrf(dest)->vrf_id; + + qpb_address_family_set(&msg->address_family, rib_dest_af(dest)); + + /* + * XXX Hardcode subaddress family for now. + */ + msg->sub_address_family = QPB__SUB_ADDRESS_FAMILY__UNICAST; + msg->key = fpm_route_key_create (allocator, rib_dest_prefix(dest)); + if (!msg->key) { + assert(0); + return NULL; + } + + return msg; +} + +/* + * add_nexthop + */ +static inline int +add_nexthop (qpb_allocator_t *allocator, Fpm__AddRoute *msg, rib_dest_t *dest, + struct nexthop *nexthop) +{ + uint32_t if_index; + union g_addr *gateway, *src; + + gateway = src = NULL; + + if_index = nexthop->ifindex; + + if (nexthop->type == NEXTHOP_TYPE_IPV4 + || nexthop->type == NEXTHOP_TYPE_IPV4_IFINDEX) + { + gateway = &nexthop->gate; + if (nexthop->src.ipv4.s_addr) + src = &nexthop->src; + } + + if (nexthop->type == NEXTHOP_TYPE_IPV6 + || nexthop->type == NEXTHOP_TYPE_IPV6_IFNAME + || nexthop->type == NEXTHOP_TYPE_IPV6_IFINDEX) + { + gateway = &nexthop->gate; + } + + if (nexthop->type == NEXTHOP_TYPE_IFINDEX + || nexthop->type == NEXTHOP_TYPE_IFNAME) + { + if (nexthop->src.ipv4.s_addr) + src = &nexthop->src; + } + + if (!gateway && if_index == 0) + return 0; + + /* + * We have a valid nexthop. + */ + { + Fpm__Nexthop *pb_nh; + pb_nh = QPB_ALLOC(allocator, typeof(*pb_nh)); + if (!pb_nh) { + assert(0); + return 0; + } + + fpm__nexthop__init(pb_nh); + + if (if_index != 0) { + pb_nh->if_id = qpb_if_identifier_create (allocator, if_index); + } + + if (gateway) { + pb_nh->address = qpb_l3_address_create (allocator, gateway, + rib_dest_af(dest)); + } + + msg->nexthops[msg->n_nexthops++] = pb_nh; + } + + // TODO: Use src. + + return 1; +} + +/* + * create_add_route_message + */ +static Fpm__AddRoute * +create_add_route_message (qpb_allocator_t *allocator, rib_dest_t *dest, + struct rib *rib) +{ + Fpm__AddRoute *msg; + int discard; + struct nexthop *nexthop, *tnexthop; + int recursing; + uint num_nhs, u; + struct nexthop *nexthops[MAX (MULTIPATH_NUM, 64)]; + + msg = QPB_ALLOC(allocator, typeof(*msg)); + if (!msg) { + assert(0); + return NULL; + } + + fpm__add_route__init(msg); + + msg->vrf_id = rib_dest_vrf(dest)->vrf_id; + + qpb_address_family_set (&msg->address_family, rib_dest_af(dest)); + + /* + * XXX Hardcode subaddress family for now. + */ + msg->sub_address_family = QPB__SUB_ADDRESS_FAMILY__UNICAST; + msg->key = fpm_route_key_create (allocator, rib_dest_prefix(dest)); + qpb_protocol_set (&msg->protocol, rib->type); + + if ((rib->flags & ZEBRA_FLAG_BLACKHOLE) || (rib->flags & ZEBRA_FLAG_REJECT)) + discard = 1; + else + discard = 0; + + if (discard) + { + if (rib->flags & ZEBRA_FLAG_BLACKHOLE) { + msg->route_type = FPM__ROUTE_TYPE__BLACKHOLE; + } else if (rib->flags & ZEBRA_FLAG_REJECT) { + msg->route_type = FPM__ROUTE_TYPE__UNREACHABLE; + } else { + assert (0); + } + return msg; + } + else { + msg->route_type = FPM__ROUTE_TYPE__NORMAL; + } + + msg->metric = rib->metric; + + /* + * Figure out the set of nexthops to be added to the message. + */ + num_nhs = 0; + for (ALL_NEXTHOPS_RO (rib->nexthop, nexthop, tnexthop, recursing)) + { + if (MULTIPATH_NUM != 0 && num_nhs >= MULTIPATH_NUM) + break; + + if (num_nhs >= ZEBRA_NUM_OF(nexthops)) + break; + + if (CHECK_FLAG (nexthop->flags, NEXTHOP_FLAG_RECURSIVE)) + continue; + + if (!CHECK_FLAG (nexthop->flags, NEXTHOP_FLAG_ACTIVE)) + continue; + + nexthops[num_nhs] = nexthop; + num_nhs++; + } + + if (!num_nhs) { + zfpm_debug ("netlink_encode_route(): No useful nexthop."); + assert(0); + return NULL; + } + + /* + * And add them to the message. + */ + if (!(msg->nexthops = qpb_alloc_ptr_array(allocator, num_nhs))) { + assert(0); + return NULL; + } + + msg->n_nexthops = 0; + for (u = 0; u < num_nhs; u++) { + if (!add_nexthop(allocator, msg, dest, nexthops[u])) { + assert(0); + return NULL; + } + } + + assert(msg->n_nexthops == num_nhs); + + return msg; +} + +/* + * create_route_message + */ +static Fpm__Message * +create_route_message (qpb_allocator_t *allocator, rib_dest_t *dest, + struct rib *rib) +{ + Fpm__Message *msg; + + msg = QPB_ALLOC(allocator, typeof(*msg)); + if (!msg) { + assert(0); + return NULL; + } + + fpm__message__init(msg); + + if (!rib) { + msg->type = FPM__MESSAGE__TYPE__DELETE_ROUTE; + msg->delete_route = create_delete_route_message(allocator, dest, rib); + if (!msg->delete_route) { + assert(0); + return NULL; + } + return msg; + } + + msg->type = FPM__MESSAGE__TYPE__ADD_ROUTE; + msg->add_route = create_add_route_message(allocator, dest, rib); + if (!msg->add_route) { + assert(0); + return NULL; + } + + return msg; +} + +/* + * zfpm_protobuf_encode_route + * + * Create a protobuf message corresponding to the given route in the + * given buffer space. + * + * Returns the number of bytes written to the buffer. 0 or a negative + * value indicates an error. + */ +int +zfpm_protobuf_encode_route (rib_dest_t *dest, struct rib *rib, + uint8_t *in_buf, size_t in_buf_len) +{ + Fpm__Message *msg; + QPB_DECLARE_STACK_ALLOCATOR (allocator, 4096); + size_t len; + + QPB_INIT_STACK_ALLOCATOR (allocator); + + msg = create_route_message(&allocator, dest, rib); + if (!msg) { + assert(0); + return 0; + } + + len = fpm__message__pack(msg, (uint8_t *) in_buf); + assert(len <= in_buf_len); + + QPB_RESET_STACK_ALLOCATOR (allocator); + return len; +} -- 2.39.5