Merge pull request #2475 from LabNConsulting/working/master/no_vrf_socket_4l3mdev

Don't open per vrf sockets when net.ipv4.tcp|udp_l3mdev_accept != 0

5 files changed, 149 insertions, 15 deletions

diff --git a/bgpd/bgp_network.c b/bgpd/bgp_network.c
index 84a959d0e8..476b64e75a 100644
--- a/bgpd/bgp_network.c
+++ b/bgpd/bgp_network.c
@@ -745,7 +745,7 @@ int bgp_socket(struct bgp *bgp, unsigned short port, const char *address)
 			close(sock);
 	}
 	freeaddrinfo(ainfo_save);
-	if (count == 0) {
+	if (count == 0 && bgp->inst_type != BGP_INSTANCE_TYPE_VRF) {
 		zlog_err(
 			"%s: no usable addresses please check other programs usage of specified port %d",
 			__func__, port);
diff --git a/doc/user/bgp.rst b/doc/user/bgp.rst
index 13e0252210..0ea93a62e7 100644
--- a/doc/user/bgp.rst
+++ b/doc/user/bgp.rst
@@ -60,11 +60,11 @@ Address Families
 ----------------
 
 Multiprotocol BGP enables BGP to carry routing information for multiple Network
-Layer protocols. BGP supports multiple Address Family Identifier (AFI), namely
-IPv4 and IPv6. Support is also provided for multiple sets of per-AFI
-information via Subsequent Address Family Identifiers (SAFI). In addition to
-unicast information, VPN information :rfc:`4364` and :rfc:`4659`, and
-Encapsulation attribute :rfc:`5512` is supported.
+Layer protocols. BGP supports an Address Family Identifier (AFI) for IPv4 and
+IPv6. Support is also provided for multiple sets of per-AFI information via the
+BGP Subsequent Address Family Identifier (SAFI). FRR supports SAFIs for unicast
+information, labeled information :rfc:`3107` and :rfc:`8277`, and Layer 3 VPN
+information :rfc:`4364` and :rfc:`4659`.
 
 .. _bgp-route-selection:
 
@@ -174,6 +174,19 @@ will establish the connection with unicast only capability. When there are no
 common capabilities, FRR sends Unsupported Capability error and then resets the
 connection.
 
+.. _bgp-concepts-vrfs:
+
+VRFs: Virtual Routing and Forwarding
+------------------------------------
+
+*bgpd* supports :abbr:`L3VPN (Layer 3 Virtual Private Networks)` :abbr:`VRFs
+(Virtual Routing and Forwarding tables)` for IPv4 :rfc:`4364` and IPv6
+:rfc:`4659`.  L3VPN routes, and their associated VRF MPLS labels, can be
+distributed to VPN SAFI neighbors in the *default*, i.e., non VRF, BGP
+instance. VRF MPLS labels are reached using *core* MPLS labels which are
+distributed using LDP or BGP labeled unicast.  *bgpd* also supports inter-VRF
+route leaking.  General information on FRR's VRF support can be found in
+:ref:`zebra-vrf`.
 
 .. _bgp-router-configuration:
 
@@ -1550,10 +1563,11 @@ VRF Route Leaking
 ^^^^^^^^^^^^^^^^^
 
 BGP routes may be leaked (i.e. copied) between a unicast VRF RIB and the VPN
-SAFI RIB of the default VRF (leaking is also permitted between the unicast RIB
-of the default VRF and VPN). A shortcut syntax is also available for specifying
-leaking from one vrf to another vrf using the VPN RIB as the intemediary. A
-common application of the VPN-VRF feature is to connect a customer's private
+SAFI RIB of the default VRF for use in MPLS-based L3VPNs. Unicast routes may
+also be leaked between any VRFs (including the unicast RIB of the default BGP
+instanced). A shortcut syntax is also available for specifying leaking from one
+VRF to another VRF using the default instance's VPN RIB as the intemediary. A
+common application of the VRF-VRF feature is to connect a customer's private
 routing domain to a provider's VPN service. Leaking is configured from the
 point of view of an individual VRF: ``import`` refers to routes leaked from VPN
 to a unicast VRF, whereas ``export`` refers to routes leaked from a unicast VRF
@@ -1596,7 +1610,7 @@ auto-derived.
 General configuration
 """""""""""""""""""""
 
-Configuration of route leaking between a unicast VRF RIB and the VPN safi RIB
+Configuration of route leaking between a unicast VRF RIB and the VPN SAFI RIB
 of the default VRF is accomplished via commands in the context of a VRF
 address-family:
 
diff --git a/doc/user/installation.rst b/doc/user/installation.rst
index 7a430fdf98..26d30f1e10 100644
--- a/doc/user/installation.rst
+++ b/doc/user/installation.rst
@@ -289,6 +289,91 @@ FRR will run with any kernel configuration but some recommendations do exist.
    (:ref:`rip`) or *ospfd* (:ref:`ospfv2`) because these protocols use
    multicast.
 
+Linux sysctl settings and kernel modules
+````````````````````````````````````````
+
+There are several kernel parameters that impact overall operation of FRR when
+using Linux as a router. Generally these parameters should be set in a
+sysctl related configuration file, e.g., :file:`/etc/sysctl.conf` on
+Ubuntu based systems and a new file
+:file:`/etc/sysctl.d/90-routing-sysctl.conf` on Centos based systems.
+Additional kernel modules are also needed to support MPLS forwarding.
+
+:makevar:`IPv4 and IPv6 forwarding`
+   The following are set to enable IP forwarding in the kernel:
+
+   .. code-block:: shell
+
+      net.ipv4.conf.all.forwarding=1
+      net.ipv6.conf.all.forwarding=1
+
+:makevar:`MPLS forwarding`
+   Basic MPLS kernel support was introduced 4.1, additional capability
+   was introduced in 4.3 and 4.5. For some general information on Linux
+   MPLS support see
+   https://www.netdevconf.org/1.1/proceedings/slides/prabhu-mpls-tutorial.pdf.
+   The following modules should be loaded to support MPLS forwarding,
+   and are generally added to a configuration file such as
+   :file:`/etc/modules-load.d/modules.conf`:
+
+   .. code-block:: shell
+
+      # Load MPLS Kernel Modules
+      mpls_router
+      mpls_iptunnel
+
+   The following is an example to enable MPLS forwarding in the kernel:
+
+   .. code-block:: shell
+
+      # Enable MPLS Label processing on all interfaces
+      net.mpls.conf.eth0.input=1
+      net.mpls.conf.eth1.input=1
+      net.mpls.conf.eth2.input=1
+      net.mpls.platform_labels=100000
+
+   Make sure to add a line equal to :file:`net.mpls.conf.<if>.input` for
+   each interface *'<if>'* used with MPLS and to set labels to an
+   appropriate value.
+
+:makevar:`VRF forwarding`
+   General information on Linux VRF support can be found in 
+   https://www.kernel.org/doc/Documentation/networking/vrf.txt. Kernel
+   support for VRFs was introduced in 4.3 and improved upon through
+   4.13, which is the version most used in FRR testing (as of June
+   2018).  Additional background on using Linux VRFs and kernel specific
+   features can be found in
+   http://schd.ws/hosted_files/ossna2017/fe/vrf-tutorial-oss.pdf.
+
+   The following impacts how BGP TCP sockets are managed across VRFs:
+
+   .. code-block:: shell
+
+      net.ipv4.tcp_l3mdev_accept=0
+
+   With this setting a BGP TCP socket is opened per VRF.  This setting
+   ensures that other TCP services, such as SSH, provided for non-VRF
+   purposes are blocked from VRF associated Linux interfaces.
+
+   .. code-block:: shell
+
+      net.ipv4.tcp_l3mdev_accept=1
+
+   With this setting a single BGP TCP socket is shared across the
+   system.  This setting exposes any TCP service running on the system,
+   e.g., SSH, to all VRFs.  Generally this setting is not used in
+   environments where VRFs are used to support multiple administrative
+   groups.
+
+   **Important note** as of June 2018, Kernel versions 4.14-4.18 have a
+   known bug where VRF-specific TCP sockets are not properly handled. When
+   running these kernel versions, if unable to establish any VRF BGP
+   adjacencies, either downgrade to 4.13 or set
+   'net.ipv4.tcp_l3mdev_accept=1'. The fix for this issue is planned to be
+   included in future kernel versions so upgrading your kernel may also
+   address this issue.
+
+   
 Building
 ^^^^^^^^
 
diff --git a/doc/user/zebra.rst b/doc/user/zebra.rst
index b6060f0737..180d2d7efd 100644
--- a/doc/user/zebra.rst
+++ b/doc/user/zebra.rst
@@ -46,7 +46,7 @@ Besides the common invocation options (:ref:`common-invocation-options`), the
    ZEBRA will create an associated VRF. The other daemons will operate on the VRF
    VRF defined by *Zebra*, as usual.
 
-   .. seealso:: :ref:`vrf`
+   .. seealso:: :ref:`zebra-vrf`
 
 .. option:: --v6-rr-semantics
 
@@ -396,7 +396,7 @@ default) should the specified gateways not be reachable. E.g.:
    After setting TABLENO with this command, static routes defined after this
    are added to the specified table.
 
-.. _vrf:
+.. _zebra-vrf:
 
 Virtual Routing and Forwarding
 ==============================
diff --git a/lib/vrf.c b/lib/vrf.c
index 797b25d01d..643ad29cf8 100644
--- a/lib/vrf.c
+++ b/lib/vrf.c
@@ -121,10 +121,10 @@ int vrf_switch_to_netns(vrf_id_t vrf_id)
 
 	/* VRF is default VRF. silently ignore */
 	if (!vrf || vrf->vrf_id == VRF_DEFAULT)
-		return 0;
+		return 1;	/* 1 = default */
 	/* VRF has no NETNS backend. silently ignore */
 	if (vrf->data.l.netns_name[0] == '\0')
-		return 0;
+		return 2;	/* 2 = no netns */
 	name = ns_netns_pathname(NULL, vrf->data.l.netns_name);
 	if (debug_vrf)
 		zlog_debug("VRF_SWITCH: %s(%u)", name, vrf->vrf_id);
@@ -505,6 +505,35 @@ void vrf_terminate(void)
 	}
 }
 
+static int vrf_default_accepts_vrf(int type)
+{
+	const char *fname = NULL;
+	char buf[32] = {0x0};
+	int ret = 0;
+	FILE *fd = NULL;
+
+	/*
+	 * TCP & UDP services running in the default VRF context (ie., not bound
+	 * to any VRF device) can work across all VRF domains by enabling the
+	 * tcp_l3mdev_accept and udp_l3mdev_accept sysctl options:
+	 * sysctl -w net.ipv4.tcp_l3mdev_accept=1
+	 * sysctl -w net.ipv4.udp_l3mdev_accept=1
+	 */
+	if (type == SOCK_STREAM)
+		fname = "/proc/sys/net/ipv4/tcp_l3mdev_accept";
+	else if (type == SOCK_DGRAM)
+		fname = "/proc/sys/net/ipv4/udp_l3mdev_accept";
+	else
+		return ret;
+	fd = fopen(fname, "r");
+	if (fd == NULL)
+		return ret;
+	fgets(buf, 32, fd);
+	ret = atoi(buf);
+	fclose(fd);
+	return ret;
+}
+
 /* Create a socket for the VRF. */
 int vrf_socket(int domain, int type, int protocol, vrf_id_t vrf_id,
 	       char *interfacename)
@@ -515,6 +544,12 @@ int vrf_socket(int domain, int type, int protocol, vrf_id_t vrf_id,
 	if (ret < 0)
 		zlog_err("%s: Can't switch to VRF %u (%s)", __func__, vrf_id,
 			 safe_strerror(errno));
+	if (ret > 0 && interfacename && vrf_default_accepts_vrf(type)) {
+		zlog_err("VRF socket not used since net.ipv4.%s_l3mdev_accept != 0",
+			  (type == SOCK_STREAM ? "tcp" : "udp"));
+		errno = EEXIST; /* not sure if this is the best error... */
+		return -2;
+	}
 	ret = socket(domain, type, protocol);
 	save_errno = errno;
 	ret2 = vrf_switchback_to_initial();