<?xml version="1.0"encoding="US-ASCII"?>encoding="UTF-8"?> <!DOCTYPE rfcSYSTEM "rfc2629.dtd"[ <!ENTITYRFC7432 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.7432.xml">nbsp " "> <!ENTITYRFC7365 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.7365.xml">zwsp "​"> <!ENTITYRFC7364 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.7364.xml">nbhy "‑"> <!ENTITYRFC2119 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.2119.xml"> <!ENTITY RFC8174 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8174.xml"> <!ENTITY RFC8365 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8365.xml"> <!ENTITY I-D.ietf-nvo3-encap SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-nvo3-encap.xml"> <!ENTITY I-D.ietf-bess-evpn-lsp-ping SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-bess-evpn-lsp-ping.xml"> <!ENTITY I-D.skr-bess-evpn-pim-proxy SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.skr-bess-evpn-pim-proxy.xml"> <!ENTITY I-D.ietf-bess-evpn-optimized-ir SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-bess-evpn-optimized-ir.xml"> <!ENTITY RFC8584 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8584.xml"> <!ENTITY I-D.ietf-bess-evpn-pref-df SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-bess-evpn-pref-df.xml"> <!ENTITY I-D.ietf-bess-evpn-irb-mcast SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-bess-evpn-irb-mcast.xml"> <!ENTITY I-D.ietf-bess-evpn-ipvpn-interworking SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-bess-evpn-ipvpn-interworking.xml"> <!ENTITY RFC7348 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.7348.xml"> <!ENTITY RFC7510 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.7510.xml"> <!ENTITY RFC4364 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.4364.xml"> <!ENTITY I-D.ietf-bess-evpn-geneve SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-bess-evpn-geneve.xml"> <!ENTITY I-D.ietf-bess-evpn-mvpn-seamless-interop SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-bess-evpn-mvpn-seamless-interop.xml"> <!ENTITY I-D.sajassi-bess-secure-evpn SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.sajassi-bess-secure-evpn.xml"> <!ENTITY I-D.ietf-bess-rfc7432bis SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-bess-rfc7432bis.xml"> <!ENTITY I-D.ietf-rtgwg-bgp-pic SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-rtgwg-bgp-pic.xml"> <!ENTITY RFC5925 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.5925.xml"> <!ENTITY RFC4271 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.4271.xml"> <!ENTITY RFC4272 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.4272.xml"> <!ENTITY RFC6952 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.6952.xml"> <!ENTITY RFC8926 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8926.xml"> <!ENTITY RFC9135 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.9135.xml"> <!ENTITY RFC9136 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.9136.xml"> <!ENTITY RFC9012 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.9012.xml"> <!ENTITY RFC9161 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.9161.xml"> <!ENTITY RFC9014 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.9014.xml"> <!ENTITY RFC4760 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.4760.xml"> <!ENTITY RFC9251 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.9251.xml">wj "⁠"> ]> <rfc xmlns:xi="http://www.w3.org/2001/XInclude" submissionType="IETF" category="info" consensus="true" docName="draft-ietf-nvo3-evpn-applicability-06" number="9469" ipr="trust200902"submissionType="IETF">obsoletes="" updates="" xml:lang="en" symRefs="true" sortRefs="true" tocInclude="true" version="3"> <!-- xml2rfc v2v3 conversion 3.17.1 --> <!-- Generated by id2xml 1.5.0 on 2020-11-02T10:45:47Z --><?rfc strict="yes"?> <?rfc compact="yes"?> <?rfc subcompact="no"?> <?rfc symrefs="yes"?> <?rfc sortrefs="no"?> <?rfc text-list-symbols="o-*+"?> <?rfc toc="yes"?><front> <title abbrev="EVPN Applicability for NVO3">Applicability ofEVPNEthernet Virtual Private Network (EVPN) toNVO3Network Virtualization over Layer 3 (NVO3) Networks</title> <seriesInfo name="RFC" value="9469"/> <author fullname="Jorge Rabadan" initials="J." role="editor" surname="Rabadan"> <organization>Nokia</organization> <address> <postal> <street>520 Almanor Ave</street> <city>Sunnyvale</city> <region>CA</region> <code>94085</code><country>USA</country><country>United States of America</country> </postal> <email>jorge.rabadan@nokia.com</email> </address> </author> <author fullname="Matthew Bocci" initials="M." surname="Bocci"> <organization>Nokia</organization> <address> <email>matthew.bocci@nokia.com</email> </address> </author> <author fullname="Sami Boutros" initials="S." surname="Boutros"> <organization>Ciena</organization> <address> <email>sboutros@ciena.com</email> </address> </author> <author fullname="Ali Sajassi" initials="A." surname="Sajassi"> <organization abbrev="Cisco">Cisco Systems, Inc.</organization> <address> <email>sajassi@cisco.com</email> </address> </author> <dateday="28" month="April"month="September" year="2023"/><workgroup>NVO3 Workgroup</workgroup><area>rtg</area> <workgroup>nvo3</workgroup> <abstract><t>Ethernet<t>An Ethernet Virtual Private Network (EVPN) provides a unifiedcontrol-planecontrol plane that solves the issues of Network Virtualization Edge (NVE) auto-discovery, tenantMAC/IP disseminationMedia Access Control (MAC) / IP dissemination, and advanced features in a scablable way as required by Network VirtualizationOver Layer-3over Layer 3 (NVO3) networks. EVPN is a scalable solution for NVO3 networks and keeps the independence of the underlay IP Fabric,i.e.i.e., there is no need to enablePIMProtocol Independent Multicast (PIM) in the underlay network and maintain multicast states for tenant Broadcast Domains. This document describes the use of EVPN for NVO3networks,networks and discusses its applicability to basicLayer-2Layer 2 andLayer-3Layer 3 connectivityrequirements, as well asrequirements and to advanced features such asMAC-mobility,MAC Mobility, MAC Protection and Loop Protection,multi-homing,multihoming, Data Center Interconnect(DCI)(DCI), and much more. No new EVPN procedures areintroduced. </t>introduced.</t> </abstract> </front> <middle> <section anchor="sect-1"title="Introduction">numbered="true" toc="default"> <name>Introduction</name> <t>In Network VirtualizationOver Layer-3over Layer 3 (NVO3) networks, Network Virtualization Edge (NVE) devices(NVEs)sit at the edge of the underlay network and provideLayer-2Layer 2 andLayer-3Layer 3 connectivity among Tenant Systems (TSes) of the same tenant. The NVEs need to build and maintain mapping tables sothatthey can deliver encapsulated packets to their intended destination NVE(s). While there are different options to create and disseminate the mapping table entries, NVEs may exchange that information directly among themselves via acontrol-planecontrol plane protocol, such as Ethernet Virtual Private Network (EVPN). EVPN provides an efficient,flexibleflexible, and unifiedcontrol-planecontrol plane option that can be used forLayer-2Layer 2 andLayer-3Layer 3 Virtual Network (VN) service connectivity. This document does not introduce any new procedures in EVPN.</t> <t>In this document, we assume that the EVPNcontrol-planecontrol plane module resides in the NVEs. The NVEs can be virtual switches in hypervisors,Top Of Rack (TOR)Top-of-Rack (ToR) switches or Leafswitchesswitches, or Data Center Gateways. As described in <xreftarget="RFC7365"/>,target="RFC7365" format="default"/>, Network Virtualization Authorities (NVAs) may be used to provide the forwarding information to the NVEs, and in that case, EVPN could be used to disseminate the information across multiple federated NVAs. The applicability of EVPN would then be similar to the one described in this document. However, for simplicity, the description assumescontrol-planecontrol plane communication among NVE(s).</t> </section> <section anchor="sect-2"title="EVPNnumbered="true" toc="default"> <name>EVPN and NVO3Terminology">Terminology</name> <t>This document uses the terminology of <xreftarget="RFC7365"/>,target="RFC7365" format="default"/> in addition to the terms that follow.<list style="symbols"> <t>AC: Attachment</t> <dl spacing="normal" newline="false"> <dt>AC:</dt><dd>Attachment Circuit or logical interface associatedtowith a given BT. To determine the AC on which a packet arrived, the NVE will examine the physical/logical port and/or VLAN tags (where the VLAN tags can be individual c-tags,s-tagss-tags, or ranges ofboth).</t> <t>ARPboth).</dd> <dt>ARP andND: AddressNDP:</dt><dd>Address Resolution Protocol (IPv4) and Neighbor Discoveryprotocol (IPv6).</t> <t>BD: or Broadcast Domain, itProtocol (IPv6), respectively.</dd> <dt>BD:</dt><dd>Broadcast Domain that corresponds to a tenant IP subnet. If no suppression techniques are used, a BUM frame that is injected in a Broadcast Domain will reach all the NVEs that are attached to that Broadcast Domain. An EVI may contain one or multiple Broadcast Domains depending on the service model <xreftarget="RFC7432"/>.target="RFC7432" format="default"/>. This document will use the term Broadcast Domain to refer to a tenantsubnet.</t> <t>BT: a Bridgesubnet.</dd> <dt>BT:</dt><dd>Bridge Table, as defined in <xreftarget="RFC7432"/>.target="RFC7432" format="default"/>. A BT is the instantiation of a Broadcast Domain in an NVE. When there is a single Broadcast Domain on a given EVI, the MAC-VRF is equivalent to the BT on that NVE. Although a Broadcast Domain spans multipleNVEs,NVEs and a BT is really the instantiation of a Broadcast Domain in an NVE, this document uses BT and Broadcast Domaininterchangeably.</t> <t>BUM: Broadcast,interchangeably.</dd> <dt>BUM:</dt><dd>Broadcast, UnknownunicastUnicast, and Multicastframes.</t> <t>Clos: aframes</dd> <dt>Clos:</dt><dd>A multistage network topology described in <xreftarget="CLOS1953"/>,target="CLOS1953" format="default"/>, where all the edge switches (or Leafs) are connected to all the core switches (or Spines). Typically used in DataCenters.</t> <t>DFCenters.</dd> <dt>DF andNDF: they refer to DesignatedNDF:</dt><dd>Designated Forwarder and Non-Designated Forwarder,whichrespectively. These are the roles that a given PE can have in a givenES.</t> <t>ECMP: Equal Cost Multi-Path.</t> <t>ES: EthernetES.</dd> <dt>ECMP:</dt><dd>Equal-Cost Multipath</dd> <dt>ES:</dt><dd>Ethernet Segment. When a Tenant System (TS) is connected to one or more NVEs via a set of Ethernet links,thenthat set of links is referred to as an'Ethernet segment'."Ethernet Segment". Each ES is represented by a unique Ethernet Segment Identifier (ESI) in the NVO3networknetwork, and the ESI is used in EVPN routes that are specific to thatES.</t> <t>Ethernet Tag: UsedES.</dd> <dt>Ethernet Tag:</dt><dd>Used to represent a Broadcast Domain that is configured on a given ES for the purpose of Designated Forwarder election. Note that any of the following may be used to represent a Broadcast Domain: VIDs (including Q-in-Q tags), configured IDs,VNIs (Virtual Extensible Local Area Network (VXLAN) Network Identifiers),VNIs, normalized VIDs,I-SIDs (ServiceService InstanceIdentifiers),Identifiers (I-SIDs), etc., as long as the representation of the Broadcast Domains is configured consistently across the multihomed PEs attached to thatES.</t> <t>EVI:ES.</dd> <dt>EVI or EVPNInstance. It is a Layer-2Instance:</dt><dd>A Layer 2 Virtual Network that uses an EVPNcontrol-planecontrol plane to exchange reachability information among the member NVEs. It corresponds to a set of MAC-VRFs of the same tenant. See MAC-VRF in thissection.</t> <t>EVPN: Ethernetsection.</dd> <dt>EVPN:</dt><dd>Ethernet Virtual PrivateNetworks,Network, as described in <xreftarget="RFC7432"/>.</t> <t>EVPN VLAN-aware bundle service model: similartarget="RFC7432" format="default"/>.</dd> <dt>EVPN VLAN-Aware Bundle Service Interface:</dt><dd>Similar to the VLAN-bundlemodelinterface but each individual VLAN value is mapped to a different Broadcast Domain. In thismodelinterface, there are multiple Broadcast Domains per EVI for a given tenant. Each Broadcast Domain is identified by an "Ethernet Tag",thatwhich is acontrol-planecontrol plane value that identifies the routes for the Broadcast Domain within theEVI.</t> <t>EVPN VLAN-based service model: oneEVI.</dd> <dt>EVPN VLAN-Based Service Interface:</dt><dd>One of the three servicemodelsinterfaces defined in <xreftarget="RFC7432"/>.target="RFC7432" format="default"/>. It is characterized as a Broadcast Domain that uses a single VLAN per physical access port to attach tenant traffic to the Broadcast Domain. In this servicemodel,interface, there is only one Broadcast Domain perEVI.</t> <t>EVPN VLAN-bundle service model: similarEVI.</dd> <dt>EVPN VLAN-Bundle Service Interface:</dt><dd>Similar to the VLAN-based interface but uses a bundle of VLANs per physical port to attach tenant traffic to the Broadcast Domain.As in VLAN-based, in this modelLike the VLAN-based interface, there isa singleonly one Broadcast Domain perEVI.</t> <t>GENEVE: GenericEVI.</dd> <dt>Geneve:</dt><dd>Generic Network VirtualizationEncapsulation, anEncapsulation. An NVO3 encapsulation defined in <xreftarget="RFC8926"/>.</t> <t>IP-VRF: an IPtarget="RFC8926" format="default"/>.</dd> <dt>IP-VRF:</dt><dd>IP Virtual Routing and Forwarding table, as defined in <xreftarget="RFC4364"/>.target="RFC4364" format="default"/>. It stores IP Prefixes that are part of the tenant's IPspace,space and are distributed among NVEs of the same tenant by EVPN. A Route Distinguisher (RD) and one or more RouteTarget(s)Targets (RTs) are required properties of an IP-VRF. An IP-VRF is instantiated in an NVE for a giventenant,tenant if the NVE is attached to multiple subnets of the tenant and localinter-subnet-forwardinginter-subnet forwarding is required across thosesubnets.</t> <t>IRB: Integratedsubnets.</dd> <dt>IRB:</dt><dd>Integrated Routing andBridging interface.Bridging. It refers to the logical interface that connects a Broadcast Domain instance (or a BT) to anIP- VRFIP-VRF andallows to forwardforwards packets with a destination in a differentsubnet.</t> <t>MAC-VRF: asubnet.</dd> <dt>MAC-VRF:</dt><dd>A MAC Virtual Routing and Forwarding table, as defined in <xreftarget="RFC7432"/>.target="RFC7432" format="default"/>. The instantiation of an EVI (EVPN Instance) in an NVE. A Route Distinguisher (RD) andRoute Target(s) (RTs)one or more RTs are required properties of aMAC-VRFMAC-VRF, and they are normally different from the ones defined in the associated IP-VRF (if the MAC-VRF has an IRBinterface).</t> <t>NVE: Networkinterface).</dd> <dt>NVE:</dt><dd>Network VirtualizationEdge device, aEdge. A network entity that sits at the edge of an underlay network and implementsLayer-2Layer 2 and/orLayer-3Layer 3 network virtualization functions. The network-facing side of the NVE uses the underlyingLayer-3Layer 3 network to tunnel tenant frames to and from other NVEs. The tenant-facing side of the NVE sends and receives Ethernet frames to and from individual Tenant Systems. In this document, an NVE could be implemented as a virtual switch within a hypervisor, aswitchswitch, or a router, and it runs EVPN in thecontrol-plane.</t> <t>NVO3 tunnels: Networkcontrol plane.</dd> <dt>NVO3 tunnels:</dt><dd>Network VirtualizationOver Layer-3over Layer 3 tunnels. In this document, NVO3 tunnels refer to a way to encapsulate tenant frames or packets into IPpacketspackets, whose IP Source Addresses(SA)(SAs) or Destination Addresses(DA)(DAs) belong to the underlay IP address space, and identify NVEs connected to the same underlay network. Examples of NVO3 tunnel encapsulations are VXLAN <xreftarget="RFC7348"/>, GENEVEtarget="RFC7348" format="default"/>, Geneve <xreftarget="RFC8926"/>target="RFC8926" format="default"/>, or MPLSoUDP <xreftarget="RFC7510"/>.</t> <t>PE: Provider Edge router.</t> <t>PMSI: Provider Multicast Service Interface.</t> <t>PTA: Providertarget="RFC7510" format="default"/>.</dd> <dt>PE:</dt><dd>Provider Edge</dd> <dt>PMSI:</dt><dd>Provider Multicast ServiceInterfaceInterface</dd> <dt>PTA:</dt><dd>PMSI TunnelAttribute.</t> <t>RTAttribute</dd> <dt>RT andRD: RouteRD:</dt><dd>Route Target and RouteDistinguisher.</t> <t>RT-1,Distinguisher, respectively.</dd> <dt>RT-1, RT-2, RT-3,etc.: theyetc.:</dt><dd>These refer to the RouteTypeTypes followed by the typenumbernumbers as defined in the "EVPN Route Types" IANA registryfor EVPN route types.</t> <t>SA(see <eref target="https://www.iana.org/assignments/evpn/" brackets="angle"/>).</dd> <dt>SA andDA: SourceDA:</dt><dd>Source Address and DestinationAddress.Address, respectively. They are used along with MAC or IP,e.g.e.g., IP SA or MACDA.</t> <t>SBD: SupplementaryDA.</dd> <dt>SBD:</dt><dd>Supplementary BroadcastDomain. DefinedDomain, as defined in <xreftarget="RFC9136"/>, ittarget="RFC9136" format="default"/>. It is a Broadcast Domain that does not have any Attachment Circuits, only has IRB interfaces, and provides connectivity among all the IP-VRFs of a tenant in the Interface-ful IP-VRF-to-IP-VRFmodels.</t> <t>TS: Tenantmodels.</dd> <dt>TS:</dt><dd>Tenant System. A physical or virtual system that can play the role of a host or a forwardingelementelement, such as a router, switch, firewall, etc. It belongs to a single tenant and connects to one or more Broadcast Domains of thattenant.</t> <t>VIDs: Virtualtenant.</dd> <dt>VID:</dt><dd>Virtual Local Area NetworkIdentifiers.</t> <t>VNI: VirtualIdentifier</dd> <dt>VNI:</dt><dd>Virtual Network Identifier. Irrespective of the NVO3 encapsulation, the tunnel header always includes a VNI that is added at the ingress NVE (based on the mapping table lookup) and identifies the BT at the egress NVE. This VNI is called VNI in VXLAN orGENEVE, VSIDGeneve, Virtual Subnet ID (VSID) innvGREnvGRE, or Label in MPLSoGRE or MPLSoUDP. This documentwill referrefers to VNI as a genericVirtual Network IdentifierVNI for any NVO3encapsulation.</t> <t>VXLAN: Virtualencapsulation.</dd> <dt>VXLAN:</dt><dd>Virtual eXtensible Local AreaNetwork, anNetwork. An NVO3 encapsulation defined in <xreftarget="RFC7348"/>.</t> </list></t>target="RFC7348" format="default"/>.</dd> </dl> </section> <section anchor="sect-3"title="Whynumbered="true" toc="default"> <name>Why is EVPN Needed in NVO3Networks?">Networks?</name> <t>Data Centers have adopted NVO3 architectures mostly due to the issues discussed in <xreftarget="RFC7364"/>.target="RFC7364" format="default"/>. The architecture of a Data Center is nowadays based on a Clos design, where every Leaf is connected to a layer ofSpines,Spines and there is a number ofEqual Cost Multi-PathsECMPs between any twoleafLeaf nodes. All the links between Leaf and Spine nodes are routed links, forming what we also know as an underlay IP Fabric. The underlay IP Fabric does not have issues with loops or flooding (like old Spanning Tree Data Center designs did), convergence isfastfast, andEqual Cost Multi-PathECMP generally distributes utilization well across all the links.</t> <t>On this architecture, and as discussed by <xreftarget="RFC7364"/>,target="RFC7364" format="default"/>, multi-tenant intra-subnet and inter-subnet connectivity services are provided by NVO3 tunnels. VXLAN <xreftarget="RFC7348"/> or GENEVEtarget="RFC7348" format="default"/> and Geneve <xreftarget="RFC8926"/>target="RFC8926" format="default"/> are two examples of such NVO3 tunnels.</t> <t>Why is acontrol-planecontrol plane protocol along with NVO3 tunnels helpful? There are three main reasons:</t><t><list style="letters"> <t>Auto-discovery<ol spacing="normal" type="a"> <li>Auto-discovery of the remote NVEs that are attached to the same VPN instance(Layer-2(Layer 2 and/orLayer-3)Layer 3) as the ingress NVEis.</t> <t>Disseminationis.</li> <li>Dissemination of the MAC/IP host information so that mapping tables can be populated on the remoteNVEs.</t> <t>AdvancedNVEs.</li> <li>Advanced features such as MAC Mobility, MAC Protection, BUM and ARP/ND traffic reduction/suppression,Multi-homing,multihoming, functionality similar to Prefix Independent Convergence (PIC)like functionality<xreftarget="I-D.ietf-rtgwg-bgp-pic"/>, Fast Convergence, etc.</t> </list></t> <t>Atarget="I-D.ietf-rtgwg-bgp-pic" format="default"/>, fast convergence, etc.</li> </ol> <t>"Flood and learn" is a possible approach to achieve points (a) and (b) above for multipoint Ethernetservices, is "flood and learn".services. "Flood and learn" refers tonot using a specific control-plane on the NVEs, but rather "flood""flooding" BUM traffic from the ingress NVE to all the egress NVEs attached to the same BroadcastDomain.Domain instead of using a specific control plane on the NVEs. The egress NVEs may then use data path source MAC address "learning" on the frames received over the NVO3 tunnels. When the destination host replies and the frames arrive at the NVE that initially flooded BUM frames, the NVE will also "learn" the source MAC address of the frame encapsulated on the NVO3 tunnel. This approach has the following drawbacks:</t><t><list style="symbols"><ul spacing="normal"> <li> <t>In order to flood a given BUM frame, the ingress NVE must know the IP addresses of the remote NVEs attached to the same Broadcast Domain. This may be done asfollows:<list style="symbols"> <t>Thefollows:</t> <ul spacing="normal"> <li>The remote tunnel IP addresses can be statically provisioned on the ingress NVE. If the ingress NVE receives a BUM frame for the Broadcast Domain on an ingress Attachment Circuit, it will do ingress replication and will send the frame to all the configured egress NVE destination IP addresses in the BroadcastDomain.</t> <t>AllDomain.</li> <li>All the NVEs attached to the same Broadcast Domain can subscribe to an underlay IPMulticast Groupmulticast group that is dedicated to that Broadcast Domain. When an ingress NVE receives a BUM frame on an ingress Attachment Circuit, it will send a single copy of the frame encapsulated into an NVO3tunnel,tunnel using the multicast address as the destination IP address of the tunnel. This solution requiresProtocol Independent Multicast (PIM)PIM in the underlay network and the association of individual Broadcast Domains to underlay IP multicastgroups.</t> </list></t> <t>"Floodgroups.</li> </ul> </li> <li>"Flood and learn" solves the issues of auto-discovery and the learning of the MAC to VNI/tunnel IP mapping on the NVEs for a given Broadcast Domain. However, it does not provide a solution for advancedfeaturesfeatures, and it does not scale well (mostly due to the need for constant flooding and the underlay PIM states that must bemaintained).</t> </list></t>maintained).</li> </ul> <t>EVPN provides a unifiedcontrol-planecontrol plane that solves the issues of NVE auto-discovery, tenant MAC/IPdisseminationdissemination, and advanced features in a scalable way andkeepingkeeps the independence of the underlay IPFabric,Fabric; i.e., there is no need to enable PIM in the underlay network and maintain multicast states for tenant Broadcast Domains.</t> <t><xreftarget="sect-4"/>target="sect-4" format="default"/> describes how EVPN can be used to meet thecontrol-planecontrol plane requirements in an NVO3 network.</t> </section> <section anchor="sect-4"title="Applicabilitynumbered="true" toc="default"> <name>Applicability of EVPN to NVO3Networks">Networks</name> <t>This section discusses the applicability of EVPN to NVO3 networks. The intent is not to provide a comprehensive explanation of the protocolitselfitself, but to give an introduction and point at the corresponding referencedocument,document sothatthe reader can easily find more details if needed.</t> <section anchor="sect-4.1"title="EVPNnumbered="true" toc="default"> <name>EVPN Route Types Used in NVO3Networks">Networks</name> <t>EVPN supports multiple RouteTypesTypes, and each type has a different function. For convenience, <xreftarget="tab-evpn-route-types"/>target="tab-evpn-route-types" format="default"/> shows a summary of all the existing EVPNroute typesRoute Types andits usage.their usages. In thisdocumentdocument, we may refer to these route types as RT-x routes, where x is the type number included in the first column of <xreftarget="tab-evpn-route-types"/>.</t> <texttabletarget="tab-evpn-route-types" format="default"/>.</t> <table anchor="tab-evpn-route-types"style="all" title="EVPN route types"> <ttcol>Type</ttcol> <ttcol>Description</ttcol> <ttcol>Usage</ttcol> <c>1</c> <c>Ethernet Auto-Discovery</c> <c>Multi-homing: usedalign="center"> <name>EVPN Route Types</name> <thead> <tr> <th align="left">Type</th> <th align="left">Description</th> <th align="left">Usage</th> </tr> </thead> <tbody> <tr> <td align="left">1</td> <td align="left">Ethernet Auto-Discovery</td> <td align="left">Multihoming: Used for MAC mass-withdraw when advertised per EthernetSegment,Segment andusedfor aliasing/backup functions when advertised perEVI</c> <c>2</c> <c>MAC/IP Advertisement</c> <c>HostEVI.</td> </tr> <tr> <td align="left">2</td> <td align="left">MAC/IP Advertisement</td> <td align="left">Host MAC/IPdissemination,dissemination; supports MACmobilityMobility andprotection</c> <c>3</c> <c>Inclusiveprotection.</td> </tr> <tr> <td align="left">3</td> <td align="left">Inclusive Multicast EthernetTag</c> <c>NVETag</td> <td align="left">NVE discovery and BUM flooding treesetup</c> <c>4</c> <c>Ethernet Segment</c> <c>Multi-homing:setup.</td> </tr> <tr> <td align="left">4</td> <td align="left">Ethernet Segment</td> <td align="left">Multihoming: ES auto-discovery and DFElection</c> <c>5</c> <c>IP Prefix</c> <c>IPelection.</td> </tr> <tr> <td align="left">5</td> <td align="left">IP Prefix</td> <td align="left">IP Prefixdissemination</c> <c>6</c> <c>Selectivedissemination.</td> </tr> <tr> <td align="left">6</td> <td align="left">Selective Multicast EthernetTag</c> <c>IndicateTag</td> <td align="left">Indicate interest for a multicast S,G or*,G</c> <c>7</c> <c>Multicast*,G.</td> </tr> <tr> <td align="left">7</td> <td align="left">Multicast JoinSynch</c> <c>Multi-homing:Synch</td> <td align="left">Multihoming: S,G or *,G statesynch</c> <c>8</c> <c>Multicastsynch.</td> </tr> <tr> <td align="left">8</td> <td align="left">Multicast LeaveSynch</c> <c>Multi-homing:Synch</td> <td align="left">Multihoming: S,G or *,G leavesynch</c> <c>9</c> <c>Per-Regionsynch.</td> </tr> <tr> <td align="left">9</td> <td align="left">Per-Region I-PMSIA-D</c> <c>BUMA-D</td> <td align="left">BUM tree creation acrossregions</c> <c>10</c> <c>S-PMSI A-D</c> <c>Multicastregions.</td> </tr> <tr> <td align="left">10</td> <td align="left">S-PMSI A-D</td> <td align="left">Multicast tree for S,G or *,Gstates</c> <c>11</c> <c>Leaf A-D</c> <c>Usedstates.</td> </tr> <tr> <td align="left">11</td> <td align="left">Leaf A-D</td> <td align="left">Used for responses to explicittracking</c> </texttable>tracking.</td> </tr> </tbody> </table> </section> <section anchor="sect-4.2"title="EVPNnumbered="true" toc="default"> <name>EVPN Basic Applicability forLayer-2 Services">Layer 2 Services</name> <t>Although the applicability of EVPN to NVO3 networks spans multiple documents, EVPN's baseline specification is <xreftarget="RFC7432"/>.target="RFC7432" format="default"/>. <xreftarget="RFC7432"/>target="RFC7432" format="default"/> allows multipointlayer-2Layer 2 VPNs to be operated as IP VPNs <xreftarget="RFC4364"/> IP-VPNs,target="RFC4364" format="default"/>, where MACs and the information to set up flooding trees are distributed byMP-BGPMultiprotocol BGP (MP-BGP) <xreftarget="RFC4760"/>.target="RFC4760" format="default"/>. Based on <xreftarget="RFC7432"/>,target="RFC7432" format="default"/>, <xreftarget="RFC8365"/>target="RFC8365" format="default"/> describes how to use EVPN to deliverLayer-2Layer 2 services specifically in NVO3Networks.</t>networks.</t> <t><xreftarget="ure-evpn-for-l2-in-an-nvo3-network-example"/>target="ure-evpn-for-l2-in-an-nvo3-network-example" format="default"/> represents aLayer-2Layer 2 service deployed with an EVPN Broadcast Domain in an NVO3 network.</t> <figureanchor="ure-evpn-for-l2-in-an-nvo3-network-example" title="EVPNanchor="ure-evpn-for-l2-in-an-nvo3-network-example"> <name>EVPN for L2 in an NVO3 Network -example"> <artwork><![CDATA[Example</name> <artwork name="" type="" align="left" alt=""><![CDATA[ +--TS2---+ * | Single-Active * | ESI-1 +----+ +----+ |BD1 | |BD1 | +-------------| |--| |-----------+ | +----+ +----+ | | NVE2 NVE3 NVE4 | EVPN NVO3 Network +----+ NVE1(IP-A) | BD1|-----+ +-------------+ RT-2 | | | | | +-------+ +----+ | | +----+ | |MAC1 | NVE5 TS3 TS1--------|BD1 | | |IP1 | +----+ | MAC1 | +----+ | |Label L|---> | BD1|-----+ IP1 | | |NH IP-A| | | All-Active | Hypervisor | +-------+ +----+ ESI-2 +-------------+ | +--------------------------------------+ ]]></artwork> </figure> <t>In a simple NVO3 network, such as the example of <xreftarget="ure-evpn-for-l2-in-an-nvo3-network-example"/>,target="ure-evpn-for-l2-in-an-nvo3-network-example" format="default"/>, these are the basic constructs that EVPN uses forLayer-2Layer 2 services (orLayer-2Layer 2 Virtual Networks):</t><t><list style="symbols"> <t>BD1<ul spacing="normal"> <li>BD1 is an EVPN Broadcast Domain for a given tenant and TS1,TS2TS2, and TS3 are connected to it. The five represented NVEs are attached to BD1 and are connected to the same underlay IP network. That is, each NVE learns the remote NVEs' loopback addresses via underlay routingprotocol.</t> <t>NVE1protocol.</li> <li>NVE1 is deployed as a virtual switch in aHypervisorhypervisor with IP-A as underlay loopback IP address. The rest of the NVEs in <xreftarget="ure-evpn-for-l2-in-an-nvo3-network-example"/>target="ure-evpn-for-l2-in-an-nvo3-network-example" format="default"/> are physical switches and TS2/TS3 aremulti-homedmultihomed to them. TS1 is a virtual machine, identified by MAC1 and IP1. TS2 and TS3 are physically dual-connected toNVEs, henceNVEs; hence, they are normally not considered virtualmachines.</t> <t>Themachines.</li> <li>The terms Single-Active and All-Active in <xreftarget="ure-evpn-for-l2-in-an-nvo3-network-example"/>target="ure-evpn-for-l2-in-an-nvo3-network-example" format="default"/> refer to the mode in which the TS2 and TS3 aremulti-homedmultihomed to the NVEs in BD1. In All-Active mode, all themulti-homingmultihoming links are active and can send or receive traffic. In Single-Active mode, only one link (of the set of links connected to the NVEs) isactive.</t> </list></t>active.</li> </ul> <section anchor="sect-4.2.1"title="Auto-Discoverynumbered="true" toc="default"> <name>Auto-Discovery andAuto-Provisioning">Auto-Provisioning</name> <t>Auto-discovery is one of the basic capabilities of EVPN. The provisioning of EVPN components in NVEs is significantly automated, simplifying the deployment of services and minimizing manual operations that are prone to human error.</t> <t>These are some of theAuto-Discoveryauto-discovery andAuto-Provisioningauto-provisioning capabilities available in EVPN:</t><t><list style="symbols"><ul spacing="normal"> <li> <t>Automation on Ethernet Segments(ES): an(ESes): An Ethernet Segment is defined as a group of NVEs that are attached to the same Tenant System or network. An Ethernet Segment is identified by an Ethernet Segment Identifier (ESI) in the control plane, but neither the ESI nor the NVEs that share the same Ethernet Segment are required to be manually provisioned in the localNVE:<list style="symbols"> <t>IfNVE.</t> <ul spacing="normal"> <li>If themulti-homedmultihomed Tenant System or networkareis runningprotocolsprotocols, such asLACP (Linkthe Link Aggregation ControlProtocol)Protocol (LACP) <xreftarget="IEEE.802.1AX_2014"/>, MSTP (Multiple-instancetarget="IEEE.802.1AX_2014" format="default"/>, the Multiple Spanning TreeProtocol),Protocol (MSTP), G.8032,etc.etc., and all the NVEs in the Ethernet Segment can listen to the protocol PDUs to uniquely identify themulti-homedmultihomed Tenant System/network, then the ESI can be "auto-sensed" or "auto-provisioned" following the guidelines in <xreftarget="RFC7432"/> section 5.target="RFC7432" sectionFormat="of" section="5"/>. The ESI can also be auto-derived out of other parameters that are common to all NVEs attached to the same EthernetSegment.</t> <t>AsSegment.</li> <li>As described in <xreftarget="RFC7432"/>,target="RFC7432" format="default"/>, EVPN can also auto-derive the BGP parameters required to advertise the presence of a local Ethernet Segment in the control plane (RT and RD). Local Ethernet Segments are advertised using Ethernet Segmentroutesroutes, and the ESI-importRoute-TargetRoute Target used by Ethernet Segment routes can be auto-derived based on the procedures of <xreftarget="RFC7432"/>, section 7.6.</t> <t>Bytarget="RFC7432" sectionFormat="of" section="7.6"/>.</li> <li>By listening to other Ethernet Segment routes that match the local ESI and import Route Target, an NVE can also auto-discover the other NVEs participating in themulti-homingmultihoming for the EthernetSegment.</t> <t>OnceSegment.</li> <li>Once the NVE has auto-discovered all the NVEs attached to the same Ethernet Segment, the NVE can automatically perform the Designated ForwarderElectionelection algorithm (which determines the NVE that will forward traffic to themulti-homedmultihomed Tenant System/network). EVPN guarantees that all the NVEs in the Ethernet Segment have a consistent Designated ForwarderElection.</t> </list></t> <t>Auto-provisioningelection.</li> </ul> </li> <li>Auto-provisioning of services:whenWhen deploying aLayer-2 ServiceLayer 2 service for a tenant in an NVO3 network, all the NVEs attached to the same subnet must be configured with a MAC-VRF and the Broadcast Domain for the subnet, as well as certain parameters for them. Notethat,that if the EVPN servicemodel isinterfaces are VLAN-based or VLAN-bundle, implementations do not normally have a specific provisioning for the Broadcast Domain(sincesince, in this case, it isin that casethe same construct as theMAC-VRF).MAC-VRF. EVPN allows auto-deriving as many MAC-VRF parameters as possible. As an example, the MAC-VRF's Route Target and Route Distinguisher for the EVPN routes may be auto-derived.Section 5.1.2.1 in<xreftarget="RFC8365"/>target="RFC8365" sectionFormat="of" section="5.1.2.1"/> specifies how to auto-derive a MAC-VRF's Route Target as long as a VLAN-based servicemodelinterface is implemented. <xreftarget="RFC7432"/>target="RFC7432" format="default"/> specifies how to auto-derive the RouteDistinguisher.</t> </list></t>Distinguisher.</li> </ul> </section> <section anchor="sect-4.2.2"title="Remotenumbered="true" toc="default"> <name>Remote NVEAuto-Discovery">Auto-Discovery</name> <t>Auto-discovery via MP-BGP <xreftarget="RFC4760"/>target="RFC4760" format="default"/> is used to discover the remote NVEs attached to a given Broadcast Domain, the NVEs participating in a given redundancy group, the tunnel encapsulation types supported by an NVE, etc.</t> <t>In particular, when a new MAC-VRF and Broadcast Domain are enabled, the NVE will advertise a new Inclusive Multicast Ethernet Tag route. Besides other fields, the Inclusive Multicast Ethernet Tag route will encode the IP address of the advertising NVE, the Ethernet Tag (which is zero in the case of VLAN-based and VLAN-bundlemodels)interfaces), andalsoa PMSI Tunnel Attribute (PTA) that indicates the information about the intended way to deliver BUM traffic for the Broadcast Domain.</t><t>In<t>When BD1 is enabled in the example of <xreftarget="ure-evpn-for-l2-in-an-nvo3-network-example"/>, when BD1 is enabled,target="ure-evpn-for-l2-in-an-nvo3-network-example" format="default"/>, NVE1 will send an Inclusive Multicast Ethernet Tag route including its own IP address, an Ethernet-Tag forBD1BD1, and the PMSI Tunnel Attribute to the remote NVEs. Assuming Ingress Replication (IR) is used, the Inclusive Multicast Ethernet Tag route will include an identification for Ingress Replication in the PMSI Tunnel Attribute and theVirtual Network IdentifierVNI that the other NVEs in the Broadcast Domain must use to send BUM traffic to the advertising NVE. The other NVEs in the Broadcast Domain will import the Inclusive Multicast Ethernet Tag route and will add NVE1's IP address to the flooding list for BD1. Note that the Inclusive Multicast Ethernet Tag route is also sent with a BGP encapsulation attribute <xreftarget="RFC9012"/>target="RFC9012" format="default"/> that indicates what NVO3 encapsulation the remote NVEs should use when sending BUM traffic to NVE1.</t> <t>Refer to <xreftarget="RFC7432"/>target="RFC7432" format="default"/> for more information about the Inclusive Multicast Ethernet Tag route and forwarding of BUMtraffic, and totraffic. See <xreftarget="RFC8365"/>target="RFC8365" format="default"/> for its considerations on NVO3 networks.</t> </section> <section anchor="sect-4.2.3"title="Distributionnumbered="true" toc="default"> <name>Distribution of Tenant MAC and IPInformation">Information</name> <t>Tenant MAC/IP information is advertised to remote NVEs using MAC/IP Advertisement routes. Following the example of <xreftarget="ure-evpn-for-l2-in-an-nvo3-network-example"/>:</t> <t><list style="symbols"> <t>Intarget="ure-evpn-for-l2-in-an-nvo3-network-example" format="default"/>:</t> <ul spacing="normal"> <li>In a given EVPN Broadcast Domain,Tenant Systems'the MAC addresses of TSes are first learned at the NVE they are attachedto,to via data path or management plane learning. In <xreftarget="ure-evpn-for-l2-in-an-nvo3-network-example"/>target="ure-evpn-for-l2-in-an-nvo3-network-example" format="default"/>, we assume NVE1 learns MAC1/IP1 in the management plane (for instance, via Cloud Management System) since the NVE is a virtual switch. NVE2, NVE3,NVE4NVE4, and NVE5 areTOR/Leaf switchesToR/Leaf switches, and they normally learn MAC addresses via datapath.</t> <t>Oncepath.</li> <li>Once NVE1's BD1 learns MAC1/IP1, NVE1 advertises that information along with aVirtual Network IdentifierVNI andNext HopNext-Hop IP-A inana MAC/IP Advertisement route. The EVPN routes are advertised using the RouteDistinguisher/RouteDistinguisher / Route Targets of the MAC-VRF where the Broadcast Domain belongs. Similarly, all the NVEs in BD1 learn local MAC/IP addresses and advertise them in MAC/IP Advertisementroutes.</t> <t>Theroutes.</li> <li>The remote NVEs can then add MAC1 to their mapping table for BD1 (BT). For instance, when TS3 sends frames to NVE4 with the destination MAC address = MAC1, NVE4 does a MAC lookup on the Bridge Table that yields IP-A and Label L. NVE4 can then encapsulate the frame into an NVO3 tunnel with IP-A as the tunnel destination IP address and L as theVirtual Network Identifier.VNI. Note that the MAC/IP Advertisement route may also contain the host's IP address (as shown in the example of <xreftarget="ure-evpn-for-l2-in-an-nvo3-network-example"/>).target="ure-evpn-for-l2-in-an-nvo3-network-example" format="default"/>). While the MAC of the received MAC/IP Advertisement route is installed in the Bridge Table, the IP address may be installed in theProxy-ARP/NDProxy ARP/ND table (if enabled) or in the ARP/IP-VRF tables if the Broadcast Domain has an IRB. See <xreftarget="sect-4.7.3"/> to seetarget="sect-4.7.3" format="default"/> for more information aboutProxy-ARP/NDProxy ARP/ND and <xreftarget="sect-4.3"/>.target="sect-4.3" format="default"/> for more details about IRB andLayer-3 services.</t> </list></t>Layer 3 services.</li> </ul> <t>Refer to <xreftarget="RFC7432"/>target="RFC7432" format="default"/> and <xreftarget="RFC8365"/>target="RFC8365" format="default"/> for more information about the MAC/IP Advertisement route and the forwarding of known unicast traffic.</t> </section> </section> <section anchor="sect-4.3"title="EVPNnumbered="true" toc="default"> <name>EVPN Basic Applicability forLayer-3 Services">Layer 3 Services</name> <t><xreftarget="RFC9136"/>target="RFC9136" format="default"/> and <xreftarget="RFC9135"/>target="RFC9135" format="default"/> are the reference documents that describe how EVPN can be used forLayer-3Layer 3 services.Inter Subnet ForwardingInter-subnet forwarding in EVPN networks is implemented via IRB interfaces between Broadcast Domains and IP-VRFs. An EVPN Broadcast Domain corresponds to an IP subnet. When IP packets generated in a Broadcast Domain are destined to a different subnet (different Broadcast Domain) of the same tenant, the packets are sent to the IRB attached to the local Broadcast Domain in the source NVE. As discussed in <xreftarget="RFC9135"/>,target="RFC9135" format="default"/>, depending on how the IP packets are forwarded between the ingress NVE and the egress NVE, there are two forwarding models: Asymmetric andSymmetric model.</t>Symmetric.</t> <t>The Asymmetric model is illustrated in the example of <xreftarget="ure-evpn-for-l3-in-an-nvo3-network-asymmetric-model"/>target="ure-evpn-for-l3-in-an-nvo3-network-asymmetric-model" format="default"/>, and it requires the configuration of all the Broadcast Domains of the tenant in all the NVEs attached to the same tenant.In thatThat way, there is no need to advertise IP Prefixes between NVEs since all the NVEs are attached to all the subnets. It is calledAsymmetric"Asymmetric" because the ingress and egress NVEs do not perform the same number of lookups in the data plane. In <xreftarget="ure-evpn-for-l3-in-an-nvo3-network-asymmetric-model"/>,target="ure-evpn-for-l3-in-an-nvo3-network-asymmetric-model" format="default"/>, if TS1 and TS2 are in differentsubnets,subnets and TS1 sends IP packets to TS2, the following lookups are required in the data path: a MAC lookup(onat BD1'stable),table, an IP lookup(onat theIP-VRF) andIP-VRF, a MAC lookup(onat BD2'stable)table at the ingressNVE1NVE1, andthenonly a MAC lookup at the egress NVE. The two IP-VRFs in <xreftarget="ure-evpn-for-l3-in-an-nvo3-network-asymmetric-model"/>target="ure-evpn-for-l3-in-an-nvo3-network-asymmetric-model" format="default"/> are not connected bytunnelstunnels, and all the connectivity between the NVEs is done based on tunnels between the Broadcast Domains.</t> <figureanchor="ure-evpn-for-l3-in-an-nvo3-network-asymmetric-model" title="EVPNanchor="ure-evpn-for-l3-in-an-nvo3-network-asymmetric-model"> <name>EVPN for L3 in an NVO3 Network - Asymmetricmodel"> <artwork><![CDATA[Model</name> <artwork name="" type="" align="left" alt=""><![CDATA[ +-------------------------------------+ | EVPN NVO3 | | | NVE1 NVE2 +--------------------+ +--------------------+ | +---+IRB +------+ | | +------+IRB +---+ | TS1-----|BD1|----|IP-VRF| | | |IP-VRF|----|BD1| | | +---+ | | | | | | +---+ | | +---+ | | | | | | +---+ | | |BD2|----| | | | | |----|BD2|----TS2 | +---+IRB +------+ | | +------+IRB +---+ | +--------------------+ +--------------------+ | | +-------------------------------------+ ]]></artwork> </figure> <t>In the Symmetric model, depicted in <xreftarget="ure-evpn-for-l3-in-an-nvo3-network-symmetric-model"/>,target="ure-evpn-for-l3-in-an-nvo3-network-symmetric-model" format="default"/>, the same number of data path lookups is needed at the ingress and egress NVEs. For example, if TS1 sends IP packets to TS3, the following data path lookups are required: a MAC lookup at NVE1's BD1 table, an IP lookup at NVE1'sIP-VRFIP-VRF, andthenan IP lookup and MAC lookup at NVE2's IP-VRF andBD3BD3, respectively. In the Symmetric model, theInter Subnetinter-subnet connectivity between NVEs is done based on tunnels between the IP-VRFs.</t> <figureanchor="ure-evpn-for-l3-in-an-nvo3-network-symmetric-model" title="EVPNanchor="ure-evpn-for-l3-in-an-nvo3-network-symmetric-model"> <name>EVPN for L3 in an NVO3 Network - Symmetricmodel"> <artwork><![CDATA[Model</name> <artwork name="" type="" align="left" alt=""><![CDATA[ +-------------------------------------+ | EVPN NVO3 | | | NVE1 NVE2 +--------------------+ +--------------------+ | +---+IRB +------+ | | +------+IRB +---+ | TS1-----|BD1|----|IP-VRF| | | |IP-VRF|----|BD3|-----TS3 | +---+ | | | | | | +---+ | | +---+IRB | | | | +------+ | TS2-----|BD2|----| | | +--------------------+ | +---+ +------+ | | +--------------------+ | | | +-------------------------------------+ ]]></artwork> </figure> <t>The Symmetric model scales better than the Asymmetric model because it does not require the NVEs to be attached to all the tenant's subnets. However, it requires the use of NVO3 tunnels on the IP-VRFs and the exchange of IP Prefixes between the NVEs in the control plane. EVPN uses MAC/IP Advertisement routes for the exchange of host IP routes and IPPrefixesPrefix routes for the exchange of prefixes of anylength (includinglength, including hostroutes too).routes. As an example, in <xreftarget="ure-evpn-for-l3-in-an-nvo3-network-symmetric-model"/>,target="ure-evpn-for-l3-in-an-nvo3-network-symmetric-model" format="default"/>, NVE2 needs to advertise TS3's host route and/or TS3'ssubnet,subnet so that the IP lookup on NVE1's IP-VRF succeeds.</t> <t><xreftarget="RFC9135"/>target="RFC9135" format="default"/> specifies the use of MAC/IP Advertisement routes for the advertisement of host routes.Section 4.4.1 in<xreftarget="RFC9136"/>target="RFC9136" sectionFormat="of" section="4.4.1"/> specifies the use of IP Prefix routes for the advertisement of IP Prefixes in an "Interface-less IP-VRF-to-IP-VRF Model". The Symmetric model for host routes can be implemented following either approach:</t><t><list style="letters"> <t><xref target="RFC9135"/><ol spacing="normal" type="a"><li> <xref target="RFC9135" format="default"/> uses MAC/IP Advertisement routes to convey the information to populateLayer-2, ARP/NDLayer 2, ARP/ND, andLayer-3Layer 3 Forwarding Information Base tables in the remote NVE. For instance, in <xreftarget="ure-evpn-for-l3-in-an-nvo3-network-symmetric-model"/>,target="ure-evpn-for-l3-in-an-nvo3-network-symmetric-model" format="default"/>, NVE2 would advertise a MAC/IP Advertisement route with TS3's IP and MACaddresses,addresses andincludinginclude twolabels/Virtual Network Identifiers:labels / VNIs: a label-3/VNI-3 that identifies BD3 for MAC lookup (that would be used forLayer-2Layer 2 traffic in case NVE1 was attached to BD3 too) and a label-1/VNI-1 that identifies the IP-VRF for IP lookup(and will(that would be used forLayer-3Layer 3 traffic). NVE1 imports the MAC/IP Advertisement route and installs TS3's IP in the IP-VRF route table with label-1/VNI-1.Traffic fromTraffic, e.g., from TS2 to TS3,willwould be encapsulated with label-1/VNI-1 and forwarded toNVE2.</t> <t><xref target="RFC9136"/>NVE2.</li> <li> <xref target="RFC9136" format="default"/> uses MAC/IP Advertisement routes to convey the information to populate theLayer-2Layer 2 Forwarding InformationBase andBase, ARP/ND tables, and IP Prefix routes to populate the IP-VRFLayer-3Layer 3 Forwarding Information Base table. For instance, in <xreftarget="ure-evpn-for-l3-in-an-nvo3-network-symmetric-model"/>,target="ure-evpn-for-l3-in-an-nvo3-network-symmetric-model" format="default"/>, NVE2 would advertise a MAC/IP Advertisement route including TS3's MAC and IP addresses with a single label-3/VNI-3. In this example, this MAC/IP Advertisement route wouldn't be imported by NVE1 because NVE1 is not attached to BD3. In addition, NVE2 would advertise an IP Prefix route with TS3's IP address and label-1/VNI-1. This IP Prefix route would be imported by NVE1's IP-VRF and the host route installed in theLayer-3Layer 3 Forwarding Information Base associatedtowith label-1/VNI-1. Traffic from TS2 to TS3 would be encapsulated withlabel-1/VNI-1.</t> </list></t>label-1/VNI-1.</li> </ol> </section> <section anchor="sect-4.4"title="EVPNnumbered="true" toc="default"> <name>EVPN as Control Plane for NVO3 Encapsulations andGENEVE">Geneve</name> <t><xreftarget="RFC8365"/>target="RFC8365" format="default"/> describes how to use EVPN for NVO3 encapsulations, such us VXLAN,nvGREnvGRE, or MPLSoGRE. The procedures can be easily applicable to any other NVO3 encapsulation,in particular GENEVE.</t> <t>The Generic Network Virtualization Encapsulationparticularly Geneve.</t> <t>Geneve <xreftarget="RFC8926"/>target="RFC8926" format="default"/> is the proposed standard encapsulation specified in the IETF Network Virtualization Overlays Working Group. The EVPN control plane can signal theGENEVEGeneve encapsulation type in the BGP Tunnel Encapsulation Extended Community (see <xreftarget="RFC9012"/>).</t> <t>GENEVEtarget="RFC9012" format="default"/>).</t> <t>Geneve requires a control plane <xreftarget="I-D.ietf-nvo3-encap"/>target="I-D.ietf-nvo3-encap" format="default"/> to:</t><t><list style="numbers"> <t>Negotiate<ul spacing="normal"><li>Negotiate a subset ofGENEVEGeneve option TLVs that can be carried on aGENEVE tunnel</t> <t>EnforceGeneve tunnel,</li> <li>Enforce an order forGENEVEGeneve optionTLVs and</t> <t>LimitTLVs, and</li> <li>Limit the total number of options that could be carried on aGENEVE tunnel.</t> </list></t>Geneve tunnel.</li> </ul> <t>The EVPN control plane can easily extend the BGP Tunnel EncapsulationAttributeattribute sub-TLV <xreftarget="RFC9012"/>target="RFC9012" format="default"/> to specify theGENEVEGeneve tunnel options that can be received or transmitted over aGENEVE tunnelsGeneve tunnel by a given NVE. <xreftarget="I-D.ietf-bess-evpn-geneve"/>target="I-D.ietf-bess-evpn-geneve" format="default"/> describes the EVPN control plane extensions to supportGENEVE.</t>Geneve.</t> </section> <section anchor="sect-4.5"title="EVPNnumbered="true" toc="default"> <name>EVPN OAM and Application toNVO3">NVO3</name> <t>EVPNOAM (asOperations, Administration, and Maintenance (OAM), as described in <xreftarget="I-D.ietf-bess-evpn-lsp-ping"/>)target="I-D.ietf-bess-evpn-lsp-ping" format="default"/>, defines mechanisms to detect data plane failures in an EVPN deployment over an MPLS network. These mechanisms detect failures related toP2Ppoint-to-point (P2P) andP2MPPoint-to-Multipoint (P2MP) connectivity, for multi-tenant unicast and multicastLayer-2Layer 2 traffic, between multi-tenant access nodes connected to EVPN PE(s), and in a single-homed,single-activeSingle-Active, orall-activeAll-Active redundancy model.</t> <t>In general, EVPN OAM mechanisms defined for EVPN deployed in MPLS networks are equally applicable for EVPN in NVO3 networks.</t> </section> <section anchor="sect-4.6"title="EVPNnumbered="true" toc="default"> <name>EVPN as the Control Plane for NVO3Security">Security</name> <t>EVPN can be used to signal the security protection capabilities of a sender NVE, as well as what portion of an NVO3 packet (taking aGENEVEGeneve packet as an example) can be protected by the sender NVE, to ensure the privacy and integrity of tenant traffic carried over the NVO3 tunnels <xreftarget="I-D.sajassi-bess-secure-evpn"/>.</t>target="I-D.ietf-bess-secure-evpn" format="default"/>.</t> </section> <section anchor="sect-4.7"title="Advancednumbered="true" toc="default"> <name>Advanced EVPN Features for NVO3Networks">Networks</name> <t>This section describes how EVPN can be used to deliver advanced capabilities in NVO3 networks.</t> <section anchor="sect-4.7.1"title="Virtualnumbered="true" toc="default"> <name>Virtual Machine (VM)Mobility">Mobility</name> <t><xreftarget="RFC7432"/>target="RFC7432" format="default"/> replaces the classic EthernetFlood-and-Learn"flood and learn" behavior among NVEs with BGP-based MAClearning, which in returnlearning. In return, this provides more control over the location of MAC addresses in the Broadcast Domain and consequently advanced features, such as MAC Mobility. If we assume thatVMVirtual Machine (VM) Mobility means the VM's MAC and IP addresses move with the VM, EVPN's MAC Mobility is the required procedure that facilitates VM Mobility. According to <xreftarget="RFC7432"/> section 15,target="RFC7432" sectionFormat="of" section="15"/>, when a MAC is advertised for the first time in a Broadcast Domain, all the NVEs attached to the Broadcast Domain will store Sequence Number zero for that MAC. When the MAC "moves" to a remote NVE within the same BroadcastDomain but to a remote NVE,Domain, the NVE that just learnedlocallytheMAC,MAC locally increases the Sequence Number in the MAC/IP Advertisement route's MAC Mobility extended community to indicate that it owns the MAC now. That makes all theNVENVEs in the Broadcast Domain change their tables immediately with no need to wait for any aging timer. EVPN guarantees a fast MAC Mobility without flooding orblack-holespacket drops in the Broadcast Domain.</t> </section> <section anchor="sect-4.7.2"title="MACnumbered="true" toc="default"> <name>MAC Protection, DuplicationDetectionDetection, and LoopProtection">Protection</name> <t>The advertisement of MACs in the controlplane,plane allows advanced features such as MACprotection,Protection, DuplicationDetectionDetection, and Loop Protection.</t><t><xref target="RFC7432"/><t>In a MAC/IP Advertisement route, MAC Protection refers to EVPN's ability to indicate- in a MAC/IP Advertisement route -that a MAC must be protected by the NVE receiving theroute.route <xref target="RFC7432" format="default"/>. The Protection is indicated in the "Sticky bit" of the MAC Mobility extended community sent along the MAC/IP Advertisement route for a MAC. NVEs' Attachment Circuits that are connected to subject-to-be-protected servers orVMs,VMs may set the Sticky bit on the MAC/IP Advertisement routes sent for the MACs associatedtowith the Attachment Circuits. Also, statically configured MAC addresses should be advertised as Protected MACaddresses,addresses since they are not subject to MAC Mobility procedures.</t><t><xref target="RFC7432"/> MAC<t>MAC Duplication Detection refers to EVPN's ability to detect duplicate MACaddresses.addresses <xref target="RFC7432" format="default"/>. A "MAC move" is a relearn event that happens at an access Attachment Circuit or through a MAC/IP Advertisement route with a Sequence Number that is higher than the stored one for the MAC. When a MAC moves a number of timesN(N) within an M-second window between two NVEs, the MAC is declared asDuplicatea duplicate and the detecting NVE does not re-advertise the MAC anymore.</t> <t><xreftarget="RFC7432"/>target="RFC7432" format="default"/> provides MAC Duplication Detection, and with anextensionextension, it can protect the Broadcast Domain against loops created by backdoor links between NVEs. The same principle (based on the Sequence Number) may be extended to protect the Broadcast Domain against loops. When a MAC is detected as a duplicate, the NVE may install it as a drop-MAC and discard received frames with source MAC address or the destination MAC address matching that duplicate MAC. The MAC Duplication extension to support Loop Protection is described in <xreftarget="I-D.ietf-bess-rfc7432bis"/>, section 15.3.</t>target="I-D.ietf-bess-rfc7432bis" sectionFormat="of" section="15.3"/>.</t> </section> <section anchor="sect-4.7.3"title="Reduction/Optimizationnumbered="true" toc="default"> <name>Reduction/Optimization of BUM Traffic inLayer-2 Services">Layer 2 Services</name> <t>In Broadcast Domains with a significant amount of flooding due to UnknownunicastUnicast andBroadcastbroadcast frames, EVPN may help reduce and sometimes even suppress the flooding.</t> <t>In Broadcast Domains where most of theBroadcastbroadcast traffic is caused byARP (Addressthe Address ResolutionProtocol)Protocol (ARP) andND (Neighbor Discovery) protocolsthe Neighbor Discovery Protocol (NDP) on the Tenant Systems, EVPN'sProxy-ARPProxy ARP andProxy-NDProxy ND capabilities may reduce the flooding drastically. The use ofProxy-ARP/NDProxy ARP/ND is specified in <xreftarget="RFC9161"/>.</t> <t>Proxy-ARP/ND procedurestarget="RFC9161" format="default"/>.</t> <t>Proxy ARP/ND procedures, along with the assumption that Tenant Systems always issue aGARP (Gratuitous ARP)Gratuitous ARP (GARP) or an unsolicited Neighbor Advertisement message when they come up in the Broadcast Domain, may drastically reduce theunknown unicastUnknown Unicast flooding in the Broadcast Domain.</t> <t>The flooding caused by Tenant Systems'IGMP/MLDIGMP / Multicast Listener Discovery (MLD) or PIM messages in the Broadcast Domain may also be suppressed by the use of IGMP/MLD and PIM Proxy functions, as specified in <xreftarget="RFC9251"/>target="RFC9251" format="default"/> and <xreftarget="I-D.skr-bess-evpn-pim-proxy"/>.target="I-D.skr-bess-evpn-pim-proxy" format="default"/>. These two documents also specify how to forward IP multicast traffic efficiently within the same Broadcast Domain, translate soft state IGMP/MLD/PIM messages into hard state BGProutesroutes, and providefast-convergencefast convergence redundancy for IPMulticastmulticast onmulti-homed Ethernet Segments (ESes).</t>multihomed ESes.</t> </section> <section anchor="sect-4.7.4"title="Ingressnumbered="true" toc="default"> <name>Ingress Replication (IR) Optimization for BUMTraffic">Traffic</name> <t>When an NVE attached to a given Broadcast Domain needs to send BUM traffic for the Broadcast Domain to the remote NVEs attached to the same Broadcast Domain, Ingress Replication is a very common option in NVO3networks,networks since it is completely independent of the multicast capabilities of the underlay network. Also, if the optimization procedures to reduce/suppress the flooding in the Broadcast Domain are enabled (<xreftarget="sect-4.7.3"/>),target="sect-4.7.3" format="default"/>) in spite of creating multiple copies of the same frame at the ingress NVE, Ingress Replication may be good enough. However, in Broadcast Domains where Multicast (or Broadcast) traffic is significant, Ingress Replication may be very inefficient and cause performance issues onvirtual-switch-basedvirtual switch-based NVEs.</t> <t><xreftarget="I-D.ietf-bess-evpn-optimized-ir"/>target="I-D.ietf-bess-evpn-optimized-ir" format="default"/> specifies the use ofAR (Assisted Replication)Assisted Replication (AR) NVO3 tunnels in EVPN Broadcast Domains. AR retains the independence of the underlay network while providing a way to forward Broadcast andMulticastmulticast traffic efficiently. AR uses AR-REPLICATORs that can replicate theBroadcast/Multicastbroadcast/multicast traffic on behalf of the AR-LEAF NVEs. The AR-LEAF NVEs are typicallyvirtual-switchesvirtual switches or NVEs with limited replication capabilities. AR can work in a single-stage replication mode (Non-Selective Mode) or in a dual-stage replication mode (Selective Mode). Both modes are detailed in <xreftarget="I-D.ietf-bess-evpn-optimized-ir"/>.</t>target="I-D.ietf-bess-evpn-optimized-ir" format="default"/>.</t> <t>In addition, <xreftarget="I-D.ietf-bess-evpn-optimized-ir"/> alsotarget="I-D.ietf-bess-evpn-optimized-ir" format="default"/> describes a procedure to avoid sendingBroadcast, Multicast or Unknown unicastBUM to certain NVEs that do not need that type of traffic. This is done by enablingPFL (PrunedPruned FloodLists)Lists (PFLs) on a given Broadcast Domain. For instance, avirtual-switchvirtual switch NVE that learns all its local MAC addresses for a Broadcast Domain via a Cloud ManagementSystem,System does not need to receive the Broadcast Domain's UnknownunicastUnicast traffic.Pruned Flood ListsPFLs help optimize the BUM flooding in the Broadcast Domain.</t> </section> <section anchor="sect-4.7.5"title="EVPN Multi-Homing">numbered="true" toc="default"> <name>EVPN Multihoming</name> <t>Another fundamental concept in EVPN ismulti-homing.multihoming. A given Tenant System can bemulti-homedmultihomed to two or more NVEs for a given Broadcast Domain, and the set of links connected to the same Tenant System is defined asEthernet Segment (ES).an ES. EVPN supportssingle-activeSingle-Active andall-active multi-homing.All-Active multihoming. Insingle-active multi-homingSingle-Active multihoming, only one link in the Ethernet Segment is active. Inall-active multi-homingAll-Active multihoming, all the links in the Ethernet Segment are active for unicast traffic. Both modes support load-balancing:</t><t><list style="symbols"> <t>Single-active multi-homing<ul spacing="normal"> <li>Single-Active multihoming means per-service load-balancing to/from the Tenant System. For example, in <xreftarget="ure-evpn-for-l2-in-an-nvo3-network-example"/>,target="ure-evpn-for-l2-in-an-nvo3-network-example" format="default"/> for BD1, only one of the NVEs can forward traffic from/to TS2. For a different Broadcast Domain, the other NVE may forwardtraffic.</t> <t>All-active multi-homingtraffic.</li> <li>All-active multihoming means per-flowload-balandingload-balancing for unicast frames to/from the Tenant System. That is, in <xreftarget="ure-evpn-for-l2-in-an-nvo3-network-example"/>target="ure-evpn-for-l2-in-an-nvo3-network-example" format="default"/> and for BD1, both NVE4 and NVE5 can forward known unicast traffic to/from TS3. For BUMtraffictraffic, only one of the two NVEs can forward traffic to TS3, and both can forward traffic fromTS3.</t> </list>ThereTS3.</li> </ul> <t>There are two key aspects in the EVPNmulti-homingmultihoming procedures:</t><t><list style="symbols"> <t>DF (Designated Forwarder) election: the<dl spacing="normal" newline="true"> <dt>Designated Forwarder (DF) election:</dt><dd>The Designated Forwarder is the NVE that forwards the traffic to the Ethernet Segment insingle-activeSingle-Active mode. In the case ofall-active,All-Active mode, the Designated Forwarder is the NVE that forwards the BUM traffic to the EthernetSegment.</t> <t>Split-horizon function: preventsSegment.</dd> <dt>Split-horizon function:</dt><dd>Prevents the Tenant System from receiving echoed BUM frames that the Tenant System itself sent to the Ethernet Segment. This is especially relevant inall-active Ethernet Segments,All-Active ESes where theTenant SystemTS may forward BUM frames to anon-DesignatedNon-Designated Forwarder NVE that can flood the BUM frames back to the Designated Forwarder NVE and then back to theTenant System.TS. As an example,in <xref target="ure-evpn-for-l2-in-an-nvo3-network-example"/>,assuming NVE4 is the Designated Forwarder for ESI-2 in BD1, <xref target="ure-evpn-for-l2-in-an-nvo3-network-example" format="default"/> shows that BUM frames sent from TS3 to NVE5 will be received at NVE4. NVE4and,will forward them back to TS3 since NVE4 is the Designated Forwarder forBD1, it will forward them back to TS3.BD1. Split-horizon allows NVE4 (and anymulti-homedmultihomed NVE for that matter) to identify if an EVPN BUM frame is coming from the same Ethernet Segment ordifferent, and ifa different one. If the frame belongs to the same ESI-2, NVE4 will not forward the BUM frame toTS3,TS3 in spite of being the DesignatedForwarder.</t> </list></t>Forwarder.</dd> </dl> <t>While <xreftarget="RFC7432"/>target="RFC7432" format="default"/> describes the default algorithm for the Designated ForwarderElection,election, <xreftarget="RFC8584"/>target="RFC8584" format="default"/> and <xreftarget="I-D.ietf-bess-evpn-pref-df"/>target="I-D.ietf-bess-evpn-pref-df" format="default"/> specify other algorithms and procedures that optimize the Designated ForwarderElection.</t>election.</t> <t>TheSplit-horizonsplit-horizon function is specified in <xreftarget="RFC7432"/>target="RFC7432" format="default"/>, and it is carried out by using a special ESI-label that it identifies in the datapath,path with all the BUM framesbeing originatedoriginating from a given NVE and Ethernet Segment. Since the ESI-label is an MPLS label, it cannot be used in all the non-MPLS NVO3encapsulations, thereforeencapsulations. Therefore, <xreftarget="RFC8365"/>target="RFC8365" format="default"/> defines a modifiedSplit-horizonsplit-horizon procedure that is based on the source IP address of the NVO3tunnel, andtunnel; it is known as "Local-Bias". It is worth noting that Local-Bias only works forall-active multi-homing,All-Active multihoming, and not forsingle-active multi-homing.</t>Single-Active multihoming.</t> </section> <section anchor="sect-4.7.6"title="EVPNnumbered="true" toc="default"> <name>EVPN Recursive Resolution forInter-SubnetInter-subnet UnicastForwarding">Forwarding</name> <t><xreftarget="sect-4.3"/>target="sect-4.3" format="default"/> describes how EVPN can be used forInter Subnet Forwardinginter-subnet forwarding among subnets of the same tenant. MAC/IP Advertisement routes and IP Prefix routes allow the advertisement of host routes and IP Prefixes (IP Prefix route) of any length. The procedures outlined by <xreftarget="sect-4.3"/>target="sect-4.3" format="default"/> are similar to the ones in <xreftarget="RFC4364"/>,target="RFC4364" format="default"/>, but they are only for NVO3 tunnels. However, <xreftarget="RFC9136"/>target="RFC9136" format="default"/> also defines advancedInter Subnet Forwardinginter-subnet forwarding procedures that allow the resolution of IP Prefix routestonot only to BGPnext-hopsnext hops but also to "overlay indexes" that can be a MAC, a Gateway IP(GW-IP)(GW-IP), or an ESI, all of them in the tenant space.</t> <t><xreftarget="ure-evpn-for-l3-recursive-resolution-example"/>target="ure-evpn-for-l3-recursive-resolution-example" format="default"/> illustrates an example that uses Recursive Resolution to a GW-IP as per <xreftarget="RFC9136"/> section 4.4.2.target="RFC9136" sectionFormat="of" section="4.4.2"/>. In this example, IP-VRFs in NVE1 and NVE2 are connected byan SBD (Supplementarya Supplementary BroadcastDomain).Domain (SBD). An SBD is a Broadcast Domain that connects all the IP-VRFs of the sametenant,tenant viaIRB,IRB and has no Attachment Circuits. NVE1 advertises the host route TS2-IP/L (IP address and Prefix Length of TS2) in an IP Prefix route with overlay index GW-IP=IP1. Also, IP1 is advertised inana MAC/IP Advertisement route associatedtowith M1,VNI-SVNI-S, and BGP next-hop NVE1. Upon importing the two routes, NVE2 installs TS2-IP/L in the IP-VRF with anext-hopnext hop that is the GW-IP IP1. NVE2 also installs M1 in the Supplementary Broadcast Domain, with VNI-S and NVE1 asnext-hop.next hop. If TS3 sends a packet with IP DA=TS2, NVE2 will perform a Recursive Resolution of the IP Prefix route prefix information to the forwarding information of the correlated MAC/IP Advertisement route. The IP Prefix route's Recursive Resolution has severaladvantagesadvantages, such as better convergence in scaled networks (since multiple IP Prefix routes can be invalidated with a single withdrawal of the overlay index route) or the ability to advertise multiple IP Prefix routes from an overlay index that can move or change dynamically. <xreftarget="RFC9136"/>target="RFC9136" format="default"/> describes a fewuse-cases.</t>use cases.</t> <figureanchor="ure-evpn-for-l3-recursive-resolution-example" title="EVPNanchor="ure-evpn-for-l3-recursive-resolution-example"> <name>EVPN for L3 - Recursive Resolutionexample"> <artwork><![CDATA[Example</name> <artwork name="" type="" align="left" alt=""><![CDATA[ +-------------------------------------+ | EVPN NVO3 | | + NVE1 NVE2 +--------------------+ +--------------------+ | +---+IRB +------+ | | +------+IRB +---+ | TS1-----|BD1|----|IP-VRF| | | |IP-VRF|----|BD3|-----TS3 | +---+ | |-(SBD)------(SBD)-| | +---+ | | +---+IRB | |IRB(IP1/M1) IRB+------+ | TS2-----|BD2|----| | | +-----------+--------+ | +---+ +------+ | | +--------------------+ | | RT-2(M1,IP1,VNI-S,NVE1)--> | | RT-5(TS2-IP/L,GW-IP=IP1)--> | +-------------------------------------+ ]]></artwork> </figure> </section> <section anchor="sect-4.7.7"title="EVPNnumbered="true" toc="default"> <name>EVPN OptimizedInter-SubnetInter-subnet MulticastForwarding">Forwarding</name> <t>The concept of the Supplementary Broadcast Domain described in <xreftarget="sect-4.7.6"/>target="sect-4.7.6" format="default"/> is also used in <xreftarget="I-D.ietf-bess-evpn-irb-mcast"/>target="I-D.ietf-bess-evpn-irb-mcast" format="default"/> for the procedures related toInter Subnet Multicast Forwardinginter-subnet multicast forwarding across Broadcast Domains of the same tenant. For instance, <xreftarget="I-D.ietf-bess-evpn-irb-mcast"/>target="I-D.ietf-bess-evpn-irb-mcast" format="default"/> allows the efficient forwarding of IP multicast traffic from any Broadcast Domain to any other Broadcast Domain (or even to the same Broadcast Domain where theSourcesource resides). The <xreftarget="I-D.ietf-bess-evpn-irb-mcast"/>target="I-D.ietf-bess-evpn-irb-mcast" format="default"/> procedures are supported along with EVPNmulti-homing,multihoming and for any tree allowed on NVO3 networks, including IR or AR. <xreftarget="I-D.ietf-bess-evpn-irb-mcast"/>target="I-D.ietf-bess-evpn-irb-mcast" format="default"/> also describes the interoperability between EVPN and other multicast technologies such asMVPN (Multicast VPN)Multicast VPN (MVPN) and PIM for inter-subnet multicast.</t> <t><xreftarget="I-D.ietf-bess-evpn-mvpn-seamless-interop"/>target="I-D.ietf-bess-evpn-mvpn-seamless-interop" format="default"/> describes another potential solution to support EVPN to MVPN interoperability.</t> </section> <section anchor="sect-4.7.8"title="Datanumbered="true" toc="default"> <name>Data Center Interconnect(DCI)">(DCI)</name> <t>TenantLayer-2Layer 2 andLayer-3Layer 3 services deployed on NVO3 networks must often be extended to remote NVO3 networks that are connected via non-NOV3 Wide Area Networks (WANs) (mostlyMPLS based Wide Area Networks).MPLS-based WANs). <xreftarget="RFC9014"/>target="RFC9014" format="default"/> defines some architectural models that can be used to interconnect NVO3 networks via MPLSWide Area Networks.</t>WANs. </t> <t>When NVO3 networks are connected by MPLSWide Area Networks,WANs, <xreftarget="RFC9014"/>target="RFC9014" format="default"/> specifies how EVPN can be usedend-to-end,end to end in spite of using a different encapsulation in theWide Area Network.WAN. <xreftarget="RFC9014"/>target="RFC9014" format="default"/> also supports the use of NVO3 or Segment Routing (encoding 32-bit or 128-bit Segment Identifiers into labels or IPv6addressesaddresses, respectively) transport tunnels in theWide Area Network.</t>WAN.</t> <t>Even if EVPN can also be used in theWide Area NetworkWAN forLayer-2Layer 2 andLayer-3Layer 3 services, there may be a need to provide a Gateway function between EVPN for NVO3 encapsulations andIPVPNIP VPN for MPLStunnels,tunnels if the operator usesIPVPNIP VPN in theWide Area Network.WAN. <xreftarget="I-D.ietf-bess-evpn-ipvpn-interworking"/>target="I-D.ietf-bess-evpn-ipvpn-interworking" format="default"/> specifies the interworking function between EVPN andIPVPNIP VPN for unicastInter Subnet Forwarding.inter-subnet forwarding. IfInter Subnet Multicast Forwardinginter-subnet multicast forwarding is also needed across anIPVPN Wide Area Network,IP VPN WAN, <xreftarget="I-D.ietf-bess-evpn-irb-mcast"/>target="I-D.ietf-bess-evpn-irb-mcast" format="default"/> describes the required interworking between EVPN andMVPN (Multicast Virtual Private Networks).</t>MVPNs.</t> </section> </section> </section> <section anchor="sect-5"title="Security Considerations">numbered="true" toc="default"> <name>Security Considerations</name> <t>This document does not introduce any new procedure or additional signaling inEVPN,EVPN and relies on the security considerations of the individual specifications used as a reference throughout the document. In particular, and as mentioned in <xreftarget="RFC7432"/>,target="RFC7432" format="default"/>, control plane and forwarding path protection are aspects to secure in any EVPNdomain,domain when applied to NVO3 networks.</t> <t><xreftarget="RFC7432"/>target="RFC7432" format="default"/> mentions security techniques such as those discussed in <xreftarget="RFC5925"/>target="RFC5925" format="default"/> to authenticate BGP messages, and those included in <xreftarget="RFC4271"/>,target="RFC4271" format="default"/>, <xreftarget="RFC4272"/>target="RFC4272" format="default"/>, and <xreftarget="RFC6952"/>target="RFC6952" format="default"/> to secure BGP are relevant for EVPN in NVO3 networks as well.</t> </section> <section anchor="sect-6"title="IANA Considerations"> <t>None.</t>numbered="true" toc="default"> <name>IANA Considerations</name> <t>This document has no IANA actions.</t> </section> </middle> <back><references title="Normative References"> &RFC7432; &RFC7365; &RFC7364;<displayreference target="I-D.ietf-nvo3-encap" to="NVO3-ENCAP"/> <displayreference target="I-D.ietf-bess-evpn-lsp-ping" to="BESS-EVPN-LSP-PING"/> <displayreference target="I-D.skr-bess-evpn-pim-proxy" to="BESS-EVPN-PIM-PROXY"/> <displayreference target="I-D.ietf-bess-evpn-pref-df" to="BESS-EVPN-PREF-DF"/> <displayreference target="I-D.ietf-bess-evpn-irb-mcast" to="BESS-EVPN-IRB-MCAST"/> <displayreference target="I-D.ietf-bess-evpn-ipvpn-interworking" to="BESS-EVPN-IPVPN-INTERWORKING"/> <displayreference target="I-D.ietf-bess-evpn-geneve" to="BESS-EVPN-GENEVE"/> <displayreference target="I-D.ietf-bess-evpn-mvpn-seamless-interop" to="BESS-EVPN-MVPN-SEAMLESS-INTEROP"/> <displayreference target="I-D.ietf-bess-secure-evpn" to="BESS-SECURE-EVPN"/> <displayreference target="I-D.ietf-bess-rfc7432bis" to="BESS-RFC7432BIS"/> <displayreference target="I-D.ietf-rtgwg-bgp-pic" to="RTGWG-BGP-PIC"/> <displayreference target="I-D.ietf-bess-evpn-optimized-ir" to="BESS-EVPN-OPTIMIZED-IR"/> <references> <name>References</name> <references> <name>Normative References</name> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.7432.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.7365.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.7364.xml"/> </references><references title="Informative References"> &RFC9136; &RFC9135; &RFC8365; &RFC8926; &I-D.ietf-nvo3-encap; &RFC9012; &I-D.ietf-bess-evpn-lsp-ping; &RFC9161; &RFC9251; &I-D.skr-bess-evpn-pim-proxy; &I-D.ietf-bess-evpn-optimized-ir; &RFC8584; &I-D.ietf-bess-evpn-pref-df; &I-D.ietf-bess-evpn-irb-mcast; &RFC9014; &I-D.ietf-bess-evpn-ipvpn-interworking; &RFC7348; &RFC7510; &RFC4364;<references> <name>Informative References</name> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9136.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9135.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.8365.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.8926.xml"/> <!-- [I-D.ietf-nvo3-encap] IESG state Publication Requested; added the long way to include Editor roles --> <referenceanchor="CLOS1953">anchor="I-D.ietf-nvo3-encap" target="https://datatracker.ietf.org/doc/html/draft-ietf-nvo3-encap-09"> <front><title>A Study<title> Network Virtualization Overlays (NVO3) Encapsulation Considerations </title> <author role="editor" initials="S." surname="Boutros" fullname="Sami Boutros"> <organization>Ciena Corporation</organization> </author> <author role="editor" initials="D." surname="Eastlake 3rd" fullname="D. Eastlake 3rd"> </author> <date month="October" day="7" year="2022"/> </front> <seriesInfo name="Internet-Draft" value="draft-ietf-nvo3-encap-09"/> </reference> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9012.xml"/> <!-- [I-D.ietf-bess-evpn-lsp-ping] in RFC Ed queue as ofNon-Blocking Switching9/18/23 --> <xi:include href="https://datatracker.ietf.org/doc/bibxml3/reference.I-D.ietf-bess-evpn-lsp-ping.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9161.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9251.xml"/> <!-- [I-D.skr-bess-evpn-pim-proxy] IESG state Expired --> <reference anchor="I-D.skr-bess-evpn-pim-proxy" target="https://datatracker.ietf.org/doc/html/draft-skr-bess-evpn-pim-proxy-01"> <front> <title>PIM Proxy in EVPN Networks</title> <authorfullname="C.role="editor" initials="J." surname="Rabadan" fullname="Jorge Rabadan"> <organization>Nokia</organization> </author> <author initials="J." surname="Kotalwar" fullname="Jayant Kotalwar"> <organization>Nokia</organization> </author> <author initials="S." surname="Sathappan" fullname="Senthil Sathappan"> <organization>Nokia</organization> </author> <author initials="Z." surname="Zhang" fullname="Zhaohui (Jeffrey) Zhang"> <organization>Juniper</organization> </author> <author initials="A." surname="Sajassi" fullname="Ali Sajassi"> <organization>Cisco</organization> </author> <date month="October" day="30" year="2017"/> </front> <seriesInfo name="Internet-Draft" value="draft-skr-bess-evpn-pim-proxy-01"/> </reference> <!-- [I-D.ietf-bess-evpn-optimized-ir] in RFC Ed queue / MISSREF*R as of 9/18/23--> <reference anchor="I-D.ietf-bess-evpn-optimized-ir" target="https://datatracker.ietf.org/doc/html/draft-ietf-bess-evpn-optimized-ir-12"> <front> <title> Optimized Ingress Replication Solution for Ethernet VPN (EVPN) </title> <author role="editor" initials="J." surname="Rabadan" fullname="Jorge Rabadan"> <organization>Nokia</organization> </author> <author initials="S." surname="Sathappan" fullname="Senthil Sathappan"> <organization>Nokia</organization> </author> <author initials="W." surname="Lin" fullname="Wen Lin"> <organization>Juniper Networks</organization> </author> <author initials="M." surname="Katiyar" fullname="Mukul Katiyar"> <organization>Versa Networks</organization> </author> <author initials="A." surname="Sajassi" fullname="Ali Sajassi"> <organization>Cisco Systems</organization> </author> <date month="January" day="25" year="2022"/> </front> <seriesInfo name="Internet-Draft" value="draft-ietf-bess-evpn-optimized-ir-12"/> </reference> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.8584.xml"/> <!-- [I-D.ietf-bess-evpn-pref-df] IESG state AD Evaluation --> <reference anchor="I-D.ietf-bess-evpn-pref-df" target="https://datatracker.ietf.org/doc/html/draft-ietf-bess-evpn-pref-df-11"> <front> <title>Preference-based EVPN DF Election</title> <author role="editor" initials="J." surname="Rabadan" fullname="Jorge Rabadan"> <organization>Nokia</organization> </author> <author initials="S." surname="Sathappan" fullname="Senthil Sathappan"> <organization>Nokia</organization> </author> <author initials="W." surname="Lin" fullname="Wen Lin"> <organization>Juniper Networks</organization> </author> <author initials="J." surname="Drake" fullname="John Drake"> <organization>Juniper Networks</organization> </author> <author initials="A." surname="Sajassi" fullname="Ali Sajassi"> <organization>Cisco Systems</organization> </author> <date month="July" day="6" year="2023"/> </front> <seriesInfo name="Internet-Draft" value="draft-ietf-bess-evpn-pref-df-11"/> </reference> <!-- [I-D.ietf-bess-evpn-irb-mcast] IESG state AD Evaluation --> <reference anchor="I-D.ietf-bess-evpn-irb-mcast" target="https://datatracker.ietf.org/doc/html/draft-ietf-bess-evpn-irb-mcast-09"> <front> <title> EVPN Optimized Inter-Subnet Multicast (OISM) Forwarding </title> <author initials="W." surname="Lin" fullname="Wen Lin"> <organization>Juniper Networks</organization> </author> <author initials="Z." surname="Zhang" fullname="Zhaohui (Jeffrey) Zhang"> <organization>Juniper Networks</organization> </author> <author initials="J." surname="Drake" fullname="John Drake"> <organization>Juniper Networks</organization> </author> <author role="editor" initials="E." surname="Rosen" fullname="Eric C. Rosen"> <organization>Juniper Networks</organization> </author> <author initials="J." surname="Rabadan" fullname="Jorge Rabadan"> <organization>Nokia</organization> </author> <author initials="A." surname="Sajassi" fullname="Ali Sajassi"> <organization>Cisco Systems</organization> </author> <date month="February" day="21" year="2023"/> </front> <seriesInfo name="Internet-Draft" value="draft-ietf-bess-evpn-irb-mcast-09"/> </reference> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.9014.xml"/> <!-- [I-D.ietf-bess-evpn-ipvpn-interworking] I-D exists --> <reference anchor="I-D.ietf-bess-evpn-ipvpn-interworking" target="https://datatracker.ietf.org/doc/html/draft-ietf-bess-evpn-ipvpn-interworking-08"> <front> <title>EVPN Interworking with IPVPN</title> <author role="editor" initials="J." surname="Rabadan" fullname="Jorge Rabadan"> <organization>Nokia</organization> </author> <author role="editor" initials="A." surname="Sajassi" fullname="Ali Sajassi"> <organization>Cisco</organization> </author> <author initials="E." surname="Rosen" fullname="Eric C. Rosen"> <organization>Individual</organization> </author> <author initials="J." surname="Drake" fullname="John Drake"> <organization>Juniper</organization> </author> <author initials="W." surname="Lin" fullname="Wen Lin"> <organization>Juniper</organization> </author> <author initials="J." surname="Uttaro" fullname="Jim Uttaro"> <organization>AT&T</organization> </author> <author initials="A." surname="Simpson" fullname="Adam Simpson"> <organization>Nokia</organization> </author> <date month="July" day="5" year="2023"/> </front> <seriesInfo name="Internet-Draft" value="draft-ietf-bess-evpn-ipvpn-interworking-08"/> </reference> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.7348.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.7510.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.4364.xml"/> <reference anchor="CLOS1953" target="https://ieeexplore.ieee.org/document/6770468"> <front> <title>A study of non-blocking switching networks</title> <author fullname="Charles Clos" initials="C." surname="Clos"><organization>The</author> <date month="March" year="1953"/> </front> <seriesInfo name="DOI" value="10.1002/j.1538-7305.1953.tb01433.x"/> <refcontent>The Bell System Technical Journal, Vol.32(2), DOI 10.1002/j.1538- 7305.1953.tb01433.x</organization>32, Issue 2</refcontent> </reference> <!-- [I-D.ietf-bess-evpn-geneve] IESG state I-D Exists --> <reference anchor="I-D.ietf-bess-evpn-geneve" target="https://datatracker.ietf.org/doc/html/draft-ietf-bess-evpn-geneve-06"> <front> <title>EVPN control plane for Geneve</title> <author role="editor" initials="S." surname="Boutros" fullname="Sami Boutros"> <organization>Ciena</organization> </author> <author initials="A." surname="Sajassi" fullname="Ali Sajassi"> <organization>Cisco Systems</organization> </author> <author initials="J." surname="Drake" fullname="John Drake"> <organization>Juniper Networks</organization> </author> <author initials="J." surname="Rabadan" fullname="Jorge Rabadan"> <organization>Nokia</organization> </author> <author initials="S." surname="Aldrin" fullname="Sam Aldrin"> <organization>Google</organization> </author> <date month="May" day="26" year="2023"/> </front> <seriesInfo name="Internet-Draft" value="draft-ietf-bess-evpn-geneve-06"/> </reference> <!-- [I-D.ietf-bess-evpn-mvpn-seamless-interop] IESG state I-D Exists --> <xi:include href="https://datatracker.ietf.org/doc/bibxml3/reference.I-D.ietf-bess-evpn-mvpn-seamless-interop.xml"/> <!-- [I-D.sajassi-bess-secure-evpn] draft-sajassi-bess-secure-evpn-06 is expired and was replaced by draft-ietf-bess-secure-evpn-00 (I-D Exists). Entered the long way as the date was wrong --> <reference anchor="I-D.ietf-bess-secure-evpn" target="https://datatracker.ietf.org/doc/html/draft-ietf-bess-secure-evpn-00"> <front> <title>Secure EVPN</title> <author initials="A." surname="Sajassi" fullname="Ali Sajassi"> <organization>Cisco</organization> </author> <author initials="A." surname="Banerjee" fullname="Ayan Banerjee"> <organization>Cisco</organization> </author> <author initials="S." surname="Thoria" fullname="Samir Thoria"> <organization>Cisco</organization> </author> <author initials="D." surname="Carrel" fullname="David Carrel"> <organization>Graphiant</organization> </author> <author initials="B." surname="Weis" fullname="Brian Weis"> <organization>Independent</organization> </author> <author initials="J." surname="Drake" fullname="John Drake"> <organization>Juniper Networks</organization> </author> <date month="June" day="20" year="2023" /> </front> <seriesInfo name="Internet-Draft" value="draft-ietf-bess-secure-evpn-00" /> </reference> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.5925.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.4271.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.4272.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.6952.xml"/> <xi:include href="https://bib.ietf.org/public/rfc/bibxml/reference.RFC.4760.xml"/> <!-- [I-D.ietf-bess-rfc7432bis] Expired --> <reference anchor="I-D.ietf-bess-rfc7432bis" target="https://datatracker.ietf.org/doc/html/draft-ietf-bess-rfc7432bis-07"> <front> <title>BGP MPLS-Based Ethernet VPN</title> <author initials="A." surname="Sajassi" fullname="Ali Sajassi"> <organization>Cisco</organization> </author> <author initials="L." surname="Burdet" fullname="Luc AndrĂ© Burdet"> <organization>Cisco</organization> </author> <author initials="J." surname="Drake" fullname="John Drake"> <organization>Juniper</organization> </author> <author initials="J." surname="Rabadan" fullname="Jorge Rabadan"> <organization>Nokia</organization> </author> <date month="March"year="1953"/>day="13" year="2023"/> </front> <seriesInfo name="Internet-Draft" value="draft-ietf-bess-rfc7432bis-07"/> </reference> <!-- [I-D.ietf-rtgwg-bgp-pic] IESG state I-D Exists --> <reference anchor="I-D.ietf-rtgwg-bgp-pic" target="https://datatracker.ietf.org/doc/html/draft-ietf-rtgwg-bgp-pic-19"> <front> <title>BGP Prefix Independent Convergence</title> <author role="editor" initials="A." surname="Bashandy" fullname="Ahmed Bashandy"> <organization>Cisco Systems</organization> </author> <author initials="C." surname="Filsfils" fullname="Clarence Filsfils"> <organization>Cisco Systems</organization> </author> <author initials="P." surname="Mohapatra" fullname="Prodosh Mohapatra"> <organization>Sproute Networks</organization> </author> <date month="April" day="1" year="2023"/> </front> <seriesInfo name="Internet-Draft" value="draft-ietf-rtgwg-bgp-pic-19"/> </reference>&I-D.ietf-bess-evpn-geneve; &I-D.ietf-bess-evpn-mvpn-seamless-interop; &I-D.sajassi-bess-secure-evpn; &RFC5925; &RFC4271; &RFC4272; &RFC6952; &RFC4760; &I-D.ietf-bess-rfc7432bis; &I-D.ietf-rtgwg-bgp-pic;<reference anchor="IEEE.802.1AX_2014"> <front> <title>IEEE Standard for Local and metropolitan area networks -- Link Aggregation</title><author fullname="IEEE"> <organization>IEEE 802.1AX-2014, DOI 10.1109/IEEESTD.2014.7055197</organization><author> <organization>IEEE</organization> </author> <dateday="24"month="December" year="2014"/> </front> <seriesInfo name="IEEE Std" value="802.1AX-2014"/> <seriesInfo name="DOI" value="10.1109/IEEESTD.2014.7055197"/> </reference> </references> </references> <sectionanchor="sect-A" title="Acknowledgments">anchor="acknowledgments" numbered="false" toc="default"> <name>Acknowledgments</name> <t>The authorswant tothankAldrin Isaac<contact fullname="Aldrin Isaac"/> for his comments.</t> </section><section anchor="sect-B" title="Contributors"/> <section anchor="sect-C" title="Authors' Addresses"/></back> </rfc>