<?xml version='1.0' encoding='utf-8'?> version="1.0" encoding="UTF-8"?>

<!DOCTYPE rfc SYSTEM "rfc2629.dtd" [
<!ENTITY RFC0768 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.0768.xml">
<!ENTITY RFC0792 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.0792.xml">
<!ENTITY RFC1112 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.1112.xml">
<!ENTITY RFC2119 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.2119.xml">
<!ENTITY RFC4443 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.4443.xml">
<!ENTITY RFC6936 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.6936.xml">
<!ENTITY RFC8126 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8126.xml">
<!ENTITY RFC8174 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8174.xml">
<!ENTITY I-D.ietf-nvo3-encap SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.draft-ietf-nvo3-encap-05.xml">
<!ENTITY I-D.ietf-nvo3-dataplane-requirements SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.draft-ietf-nvo3-dataplane-requirements-03.xml">
<!ENTITY I-D.ietf-intarea-tunnels SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml3/reference.I-D.draft-ietf-intarea-tunnels-10.xml">
<!--ENTITY IEEE.802.1Q_2014 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml6/reference.IEEE.802.1Q_2014.xml"-->
<!ENTITY RFC1191 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.1191.xml">
<!ENTITY RFC2003 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.2003.xml">
<!ENTITY RFC8200 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8200.xml">
<!ENTITY RFC2983 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.2983.xml">
<!ENTITY RFC3031 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.3031.xml">
<!ENTITY RFC3552 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.3552.xml">
<!ENTITY RFC3985 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.3985.xml">
<!ENTITY RFC4301 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.4301.xml">
<!ENTITY RFC5374 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.5374.xml">
<!ENTITY RFC6040 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.6040.xml">
<!ENTITY RFC6335 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.6335.xml">
<!ENTITY RFC6438 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.6438.xml">
<!ENTITY RFC7348 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.7348.xml">
<!ENTITY RFC7365 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.7365.xml">
<!ENTITY RFC7637 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.7637.xml">
<!ENTITY RFC8014 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8014.xml">
<!ENTITY RFC8085 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8085.xml">
<!ENTITY RFC8086 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8086.xml">
<!ENTITY RFC8201 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8201.xml">
<!ENTITY RFC8293 SYSTEM "https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8293.xml">
]> "rfc2629-xhtml.ent">

<rfc submissionType="IETF" xmlns:xi="http://www.w3.org/2001/XInclude" ipr="trust200902" docName="draft-ietf-nvo3-geneve-16" category="std"><?rfc compact="yes"?>
	<?rfc text-list-symbols="o*+-"?>
	<?rfc subcompact="no"?>
	<?rfc sortrefs="yes"?>
	<?rfc symrefs="yes"?>
	<?rfc strict="yes"?>
	<?rfc toc="yes"?> number="8926" submissionType="IETF" category="std" consensus="true" obsoletes="" updates="" xml:lang="en" sortRefs="true" symRefs="true" tocInclude="true" version="3">

  <front>
    <title abbrev="Geneve Protocol">Geneve: Generic Network Virtualization Encapsulation</title>
    <seriesInfo name="RFC" value="8926"/>
    <author fullname="Jesse Gross" initials="J." role="editor" surname="Gross">
	<organization></organization>
	<address><email>jesse@kernel.org</email>
      <organization/>
      <address>
        <email>jesse@kernel.org</email>
      </address>
    </author>
    <author fullname="Ilango Ganga" initials="I." role="editor" surname="Ganga">
      <organization abbrev="Intel">Intel Corporation</organization>
	<address><postal><street>2200
      <address>
        <postal>
          <street>2200 Mission College Blvd.</street>
	<street>Santa Clara, CA  95054</street>
	<street>USA</street>
          <city>Santa Clara</city><region>CA</region><code>95054</code>
          <country>United States of America</country>
        </postal>
        <email>ilango.s.ganga@intel.com</email>
      </address>
    </author>
    <author fullname="T. Sridhar" initials="T." role="editor" surname="Sridhar">
      <organization abbrev="VMware">VMware, Inc.</organization>
	<address><postal><street>3401
      <address>
        <postal>
          <street>3401 Hillview Ave.</street>
	<street>Palo Alto, CA  94304</street>
	<street>USA</street>
          <city>Palo Alto</city><region>CA</region><code>94304</code>
          <country>United States of America</country>
        </postal>
	<email>tsridhar@vmware.com</email>
        <email>tsridhar@utexas.edu</email>
      </address>
    </author>
    <date day="07" month="March" month="November" year="2020"/>
	<abstract><t>

<keyword>overlay</keyword>
<keyword>tunnel</keyword>
<keyword>extensible</keyword>
<keyword>variable</keyword>
<keyword>metadata</keyword>
<keyword>options</keyword>
<keyword>endpoint</keyword>
<keyword>transit</keyword>

    <abstract>

      <t>
   Network virtualization involves the cooperation of devices with a
   wide variety of capabilities such as software and hardware tunnel
   endpoints, transit fabrics, and centralized control clusters.  As a
   result of their role in tying together different elements in of the
   system, the requirements on tunnels are influenced by all of these
   components.  Flexibility  Therefore, flexibility is therefore the most important aspect of a
   tunnel
   tunneling protocol if it is to keep pace with the evolution of the
   system. technology.
   This document describes Geneve, an encapsulation protocol designed to
   recognize and accommodate these changing capabilities and needs.</t>
    </abstract>
  </front>
  <middle>
    <section title="Introduction" anchor="section-1"><t> anchor="sec-1" numbered="true" toc="default">
      <name>Introduction</name>
      <t>
   Networking has long featured a variety of tunneling, tagging, and
   other encapsulation mechanisms.  However, the advent of network
   virtualization has caused a surge of renewed interest and a
   corresponding increase in the introduction of new protocols.  The
   large number of protocols in this space, space -- for example, ranging all the way from
   VLANs <xref target="IEEE.802.1Q_2018"/> target="IEEE.802.1Q_2018" format="default"/> and MPLS <xref target="RFC3031"/> target="RFC3031" format="default"/> through the more recent
   VXLAN <xref target="RFC7348"/> (Virtual eXtensible Local Area Network)  <xref target="RFC7348" format="default"/>
   and NVGRE <xref target="RFC7637"/> (Network Virtualization
   Using Generic Routing Encapsulation), Encapsulation) <xref target="RFC7637"
   format="default"/> -- often
   leads to questions about the need for new encapsulation formats and
   what it is about network virtualization in particular that leads to
   their proliferation. Note that the list of protocols presented above is non-exhaustive.</t>
      <t>
   While many encapsulation protocols seek to simply partition the
   underlay network or bridge between two domains, network
   virtualization views the transit network as providing connectivity
   between multiple components of a distributed system.  In many ways ways,
   this system is similar to a chassis switch with the IP underlay
   network playing the role of the backplane and tunnel endpoints on the
   edge as line cards.  When viewed in this light, the requirements
   placed on the tunnel tunneling protocol are significantly different in terms of
   the quantity of metadata necessary and the role of transit nodes.</t>
      <t>
   Work such as <xref target="VL2"/> (A "VL2: A Scalable and Flexible Data Center Network) Network" <xref target="VL2" format="default"/> and the NVO3 "NVO3 Data Plane Requirements Requirements" <xref target="I-D.ietf-nvo3-dataplane-requirements"/> target="I-D.ietf-nvo3-dataplane-requirements" format="default"/>
   have described some of the properties that the data plane must have to support network
   virtualization.  However, one additional defining requirement is the
   need to carry metadata (e.g. (e.g., system state) along with the packet data;
   example use cases of metadata are noted below. The use of
   some metadata is certainly not a foreign concept - -- nearly all
   protocols used for network virtualization have at least 24 bits of identifier
   space as a way to partition between tenants.  This is often described
   as overcoming the limits of 12-bit VLANs, and VLANs; when seen in that
   context,
   context or any context where it is a true tenant identifier, 16
   million possible entries is a large number.  However, the reality is
   that the metadata is not exclusively used to identify tenants tenants, and
   encoding other information quickly starts to crowd the space.  In
   fact, when compared to the tags used to exchange metadata between
   line cards on a chassis switch, 24-bit identifiers start to look
   quite small.  There are nearly endless uses for this metadata,
   ranging from storing input port identifiers for simple security policies to
   sending service based service-based context for advanced middlebox applications
   that terminate and re-encapsulate Geneve traffic.</t>
      <t>
   Existing tunnel tunneling protocols have each attempted to solve different
   aspects of these new requirements, requirements only to be quickly rendered out of
   date by changing control plane implementations and advancements.
   Furthermore, software and hardware components and controllers all
   have different advantages and rates of evolution - -- a fact that should
   be viewed as a benefit, not a liability or limitation.  This draft document describes Geneve, a protocol which that seeks to avoid these problems by
   providing a framework for tunneling for network virtualization rather
   than being prescriptive about the entire system.</t>
      <section title="Requirements Language" anchor="section-1.1"><t> anchor="sec-1.1" numbered="true" toc="default">
        <name>Requirements Language</name>
        <t>
   The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT",
   "SHOULD", "SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED", "MAY", "<bcp14>MUST</bcp14>", "<bcp14>MUST NOT</bcp14>", "<bcp14>REQUIRED</bcp14>", "<bcp14>SHALL</bcp14>", "<bcp14>SHALL NOT</bcp14>",
   "<bcp14>SHOULD</bcp14>", "<bcp14>SHOULD NOT</bcp14>", "<bcp14>RECOMMENDED</bcp14>", "<bcp14>NOT RECOMMENDED</bcp14>", "<bcp14>MAY</bcp14>", and "OPTIONAL" "<bcp14>OPTIONAL</bcp14>" in this
   document are to be interpreted as described in BCP 14 <xref target="RFC2119"/> target="RFC2119" format="default"/>
          <xref target="RFC8174"/> target="RFC8174" format="default"/> when, and only when, they appear in all
   capitals, as shown here.</t>
      </section>
      <section title="Terminology" anchor="section-1.2"><t> anchor="sec-1.2" numbered="true" toc="default">
        <name>Terminology</name>
        <t>
   The NVO3 Network
   Virtualization over Layer 3 (NVO3) Framework <xref target="RFC7365"/> target="RFC7365" format="default"/> defines many of the concepts commonly
   used in network virtualization.  In addition, the following terms are
   specifically meaningful in this document:</t>

	<t>
   Checksum offload.  An
<dl newline="false" spacing="normal">
 <dt>Checksum offload:</dt>
<dd>An optimization implemented by many NICs (Network Interface Controller) which Controllers)
that enables computation and verification of upper layer upper-layer protocol
   checksums in hardware on transmit and receive, respectively.  This
   typically includes IP and TCP/UDP checksums which that would otherwise be
   computed by the protocol stack in software.</t>

	<t>
   Clos network.  A software.</dd>

<dt>Clos network:</dt>  <dd>A technique for composing network fabrics larger than
   a single switch while maintaining non-blocking bandwidth across
   connection points.  ECMP is used to divide traffic across the
   multiple links and switches that constitute the fabric.  Sometimes
   termed "leaf and spine" or "fat tree" topologies.</t>

	<t>
   ECMP.  Equal topologies.</dd>

<dt>ECMP:</dt>
<dd>Equal Cost Multipath. A routing mechanism for selecting from
   among multiple best next hop next-hop paths by hashing packet headers in order
   to better utilize network bandwidth while avoiding reordering of packets
   within a flow.</t>

	<t>
   Geneve.  Generic flow.</dd>

<dt>Geneve:</dt><dd>Generic Network Virtualization Encapsulation. The tunnel tunneling
   protocol described in this document.</t>

	<t>
   LRO.  Large document.</dd>

<dt>LRO:</dt><dd>Large Receive Offload. The receive-side receiver-side equivalent function
of LSO, in which multiple protocol segments (primarily TCP) are coalesced into
 larger data units.</t>

	<t>
   LSO. units.</dd>

<dt>LSO:</dt><dd> Large Segmentation Offload. A function provided by many
   commercial NICs that allows data units larger than the MTU to be
   passed to the NIC to improve performance, the NIC being responsible
   for creating smaller segments of a size less than or equal to the MTU
   with correct protocol headers.  When referring specifically to TCP/
   IP, TCP/IP, this
   feature is often known as TSO (TCP Segmentation Offload).</t>

	<t>
   Middlebox.  The term middlebox in Offload).</dd>
        <dt>
   Middlebox:</dt><dd>  In the context of this document document, the term "middlebox" refers to network
   service functions or appliances for service interposition appliances that would typically implement NVE tunnel endpoint functionality, which terminate or re-encapsulate terminating and re-encapsulating Geneve traffic.</t>

	<t>
   NIC.  Network traffic.</dd>
        <dt>NIC:</dt><dd>Network Interface Controller. Also called as Network "Network Interface Card Card" or Network Adapter. "Network Adapter".
   A NIC could be part of a tunnel endpoint or transit device and can either
   process Geneve packets or aid in the processing of Geneve packets.</t>

	<t> packets.</dd>
        <dt>
   Transit device. device:</dt> <dd> A forwarding element (e.g. (e.g., router or switch) along the path of the tunnel
   making up part of the Underlay Network. underlay network.  A transit device may be
   capable of understanding the Geneve packet format but does not
   originate or terminate Geneve packets.</t>

	<t> packets.</dd>
        <dt>
   Tunnel endpoint. endpoint:</dt><dd>  A component performing encapsulation and
   decapsulation of packets, such as Ethernet frames or IP datagrams, in
   Geneve headers.  As the ultimate consumer of any tunnel metadata,
   tunnel endpoints have the highest level of requirements for parsing and
   interpreting tunnel headers.  Tunnel endpoints may consist of either
   software or hardware implementations or a combination of the two.
   Tunnel endpoints are frequently a component of an NVE (Network a Network Virtualization Edge) Edge (NVE)
   but may also be found in middleboxes or other elements making up an NVO3 Network.</t>

	<t>
   VM.  Virtual Machine.</t> network.</dd>
        <dt>VM:</dt><dd>Virtual Machine.</dd>
</dl>
      </section>
    </section>
    <section title="Design Requirements" anchor="section-2"><t> anchor="sec-2" numbered="true" toc="default">
      <name>Design Requirements</name>
      <t>
   Geneve is designed to support network virtualization use cases for data center environments, where environments.  In these situations,
   tunnels are typically established to act as a backplane between the
   virtual switches residing in hypervisors, physical switches, or
   middleboxes or other appliances.  An arbitrary IP network can be used
   as an underlay underlay, although Clos networks composed using ECMP links are a
   common choice to provide consistent bisectional bandwidth across all
   connection points. Many of the concepts of network virtualization overlays
   over Layer 3 IP networks are described in the NVO3 Framework <xref target="RFC7365"/>.
   Figure 1 target="RFC7365" format="default"/>.
   <xref target="ref-sample-geneve-deployment"/> shows an example of a
   hypervisor, top of
   rack a top-of-rack switch for connectivity to physical servers, and a WAN uplink
   connected using Geneve tunnels over a simplified Clos network.  These
   tunnels are used to encapsulate and forward frames from the attached
   components
   components, such as VMs or physical links.</t>
      <figure title="Sample anchor="ref-sample-geneve-deployment">
        <name>Sample Geneve Deployment" anchor="ref-sample-geneve-deployment"><artwork><![CDATA[ Deployment</name>
        <artwork name="" type="" align="left" alt=""><![CDATA[
  +---------------------+           +-------+  +------+
  | +--+  +-------+---+ |           |Transit|--|Top of|==Physical
  | |VM|--|       |   | | +------+ /|Router |  | Rack |==Servers
  | +--+  |Virtual|NIC|---|Top of|/ +-------+\/+------+
  | +--+  |Switch |   | | | Rack |\ +-------+/\+------+
  | |VM|--|       |   | | +------+ \|Transit|  |Uplink|   WAN
  | +--+  +-------+---+ |           |Router |--|      |=========>
  +---------------------+           +-------+  +------+
         Hypervisor

              ()===================================()
                      Switch-Switch Geneve Tunnels
]]></artwork>
      </figure>

      <t>
   To support the needs of network virtualization, the tunnel tunneling protocol
   should be able to take advantage of the differing (and evolving)
   capabilities of each type of device in both the underlay and overlay
   networks.  This results in the following requirements being placed on
   the data plane tunneling protocol:</t>

	<t><list style="symbols"><t>The
      <ul spacing="normal">
        <li>The data plane is generic and extensible enough to support current
      and future control planes.</t>

	<t>Tunnel planes.</li>
        <li>Tunnel components are efficiently implementable in both hardware
      and software without restricting capabilities to the lowest common
      denominator.</t>

	<t>High
      denominator.</li>
        <li>High performance over existing IP fabrics.</t>

	</list>
	</t> fabrics is maintained.</li>
      </ul>
      <t>
   These requirements are described further in the following
   subsections.</t>
      <section title="Control anchor="sec-2.1" numbered="true" toc="default">
        <name>Control Plane Independence" anchor="section-2.1"><t> Independence</name>
        <t>
   Although some protocols for network virtualization have included a
   control plane as part of the tunnel format specification (most
   notably, VXLAN <xref target="RFC7348"/> target="RFC7348" format="default"/> prescribed a multicast learning-
   based multicast-learning-based control plane), these specifications have largely been treated
   as describing only the data format.  The VXLAN packet format has
   actually seen a wide variety of control planes built on top of it.</t>
        <t>
   There is a clear advantage in settling on a data format: most of the
   protocols are only superficially different and there is little
   advantage in duplicating effort.  However, the same cannot be said of
   control planes, which are diverse in very fundamental ways.  The case
   for standardization is also less clear given the wide variety in
   requirements, goals, and deployment scenarios.</t>
        <t>
   As a result of this reality, Geneve is a pure tunnel format
   specification that is capable of fulfilling the needs of many control
   planes by explicitly not selecting any one of them.  This
   simultaneously promotes a shared data format and reduces the
   chance of obsolescence by future control plane
   enhancements.</t>
      </section>
      <section title="Data anchor="sec-2.2" numbered="true" toc="default">
        <name>Data Plane Extensibility" anchor="section-2.2"><t> Extensibility</name>
        <t>
   Achieving the level of flexibility needed to support current and
   future control planes effectively requires an options infrastructure
   to allow new metadata types to be defined, deployed, and either
   finalized or retired.  Options also allow for differentiation of
   products by encouraging independent development in each vendor's core
   specialty, leading to an overall faster pace of advancement.  By far far,
   the most common mechanism for implementing options is Type-Length-
   Value the Type-Length-Value (TLV) format.</t>

        <t>
   It should be noted that that, while options can be used to support non-
   wirespeed non-wirespeed
   control packets, they are equally important on in data packets
   as well to segregate for segregating and direct forwarding (for directing forwarding. (For instance, the
   examples given before of input port based regarding input-port-based security policies and
   terminating/re-encapsulating service interposition both require tags
   to be placed on data
   packets). packets.)  Therefore, while it would be desirable to limit the
   extensibility to only control packets for the purposes of simplifying
   the datapath, that would not satisfy the design requirements.</t>
        <section title="Efficient Implementation" anchor="section-2.2.1"><t> anchor="sec-2.2.1" numbered="true" toc="default">
          <name>Efficient Implementation</name>
       <t>
   There is often a conflict between software flexibility and hardware
   performance that is difficult to resolve.  For a given set of
   functionality, it is obviously desirable to maximize performance.
   However, that does not mean new features that cannot be run at a desired
   speed today should be disallowed.  Therefore, for a protocol to be considered
   efficiently implementable means that implementable, it is expected to have a set of common capabilities that can
   be reasonably handled across platforms along with as well as a graceful
   mechanism to handle more advanced features in the appropriate
   situations.</t>

          <t>
   The use of a variable length variable-length header and options in a protocol often
   raises questions about whether it the protocol is truly efficiently
   implementable in hardware.  To answer this question in the context of Geneve, it is
   important to first divide "hardware" into two categories: tunnel
   endpoints and transit devices.</t>
   <t>
   Tunnel endpoints must be able to parse the variable variable-length header, including any
   options, and take action.  Since these devices are actively
   participating in the protocol, they are the most affected by Geneve.
   However, as tunnel endpoints are the ultimate consumers of the data,
   transmitters can tailor their output to the capabilities of the
   recipient.</t>

          <t>
   Transit devices may be able to interpret the options, options; however,
   as non-terminating devices, transit devices
   do not originate or terminate the Geneve packet, hence MUST NOT packet. Hence, they <bcp14>MUST NOT</bcp14> modify Geneve headers and
   MUST NOT
   <bcp14>MUST NOT</bcp14> insert or delete options, which as that is the responsibility of tunnel endpoints.
   Options, if present in the packet, MUST <bcp14>MUST</bcp14> only be generated and terminated by tunnel endpoints.
   The participation of transit devices in interpreting options is
   OPTIONAL.</t>
   <bcp14>OPTIONAL</bcp14>.</t>
          <t>
   Further, either tunnel endpoints or transit devices MAY <bcp14>MAY</bcp14> use offload
   capabilities of NICs NICs, such as checksum offload offload, to improve the
   performance of Geneve packet processing.  The presence of a Geneve
   variable length
   variable-length header should not prevent the tunnel endpoints and
   transit devices from using such offload capabilities.</t>
        </section>
      </section>
      <section title="Use anchor="sec-2.3" numbered="true" toc="default">
        <name>Use of Standard IP Fabrics" anchor="section-2.3"><t> Fabrics</name>
        <t>
   IP has clearly cemented its place as the dominant transport mechanism mechanism,
   and many techniques have evolved over time to make it robust,
   efficient, and inexpensive.  As a result, it is natural to use IP
   fabrics as a transit network for Geneve.  Fortunately, the use of IP
   encapsulation and addressing is enough to achieve the primary goal of
   delivering packets to the correct point in the network through
   standard switching and routing.</t>
        <t>
   In addition, nearly all underlay fabrics are designed to exploit
   parallelism in traffic to spread load across multiple links without
   introducing reordering in individual flows.  These equal cost
   multipathing (ECMP) ECMP techniques typically involve parsing and hashing
   the addresses and port numbers from the packet to select an outgoing
   link.  However, the use of tunnels often results in poor ECMP
   performance
   performance, as without additional knowledge of the protocol as protocol, the
   encapsulated traffic is hidden from the fabric by design design, and only
   tunnel endpoint addresses are available for hashing.</t>
        <t>
   Since it is desirable for Geneve to perform well on these existing
   fabrics, it is necessary for entropy from encapsulated packets to be
   exposed in the tunnel header.  The most common technique for this is
   to use the UDP source port, which is discussed further in
   <xref target="section-3.3"/>.</t> target="sec-3.3" format="default"/>.</t>
      </section>
    </section>
    <section title="Geneve anchor="sec-3" numbered="true" toc="default">
      <name>Geneve Encapsulation Details" anchor="section-3"><t> Details</name>
      <t>
   The Geneve packet format consists of a compact tunnel header
   encapsulated in UDP over either IPv4 or IPv6.  A small fixed tunnel
   header provides control information plus a base level of
   functionality and interoperability with a focus on simplicity.  This
   header is then followed by a set of variable variable-length options to allow for
   future innovation.  Finally, the payload consists of a protocol data
   unit of the indicated type, such as an Ethernet frame. Sections <xref target="section-3.1"/> target="sec-3.1" format="counter"/>
   and <xref target="section-3.2"/> target="sec-3.2" format="counter"/> illustrate the Geneve packet format transported (for
   example) over Ethernet along with an Ethernet payload.</t>
      <section title="Geneve anchor="sec-3.1" numbered="true" toc="default">
        <name>Geneve Packet Format Over IPv4" anchor="section-3.1">

	<figure><artwork><![CDATA[ over IPv4</name>
<figure>
<name>Geneve Packet Format over IPv4</name>
        <artwork name="" type="" align="left" alt=""><![CDATA[
    0                   1                   2                   3
    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1

Outer Ethernet Header:
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                 Outer Destination MAC Address                 |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   | Outer Destination MAC Address |   Outer Source MAC Address    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                   Outer Source MAC Address                    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |Optional Ethertype=C-Tag 802.1Q|  Outer VLAN Tag Information   |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |       Ethertype=0x0800    Ethertype = 0x0800 IPv4    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

Outer IPv4 Header:
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |Version|  IHL  |Type of Service|          Total Length         |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |         Identification        |Flags|      Fragment Offset    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |  Time to Live |Protocol=17 UDP|         Header Checksum       |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                     Outer Source IPv4 Address                 |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                   Outer Destination IPv4 Address              |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

Outer UDP Header:
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |       Source Port = xxxx      |    Dest Port = 6081 Geneve    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |           UDP Length          |        UDP Checksum           |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

Geneve Header:
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |Ver|  Opt Len  |O|C|    Rsvd.  |          Protocol Type        |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |        Virtual Network Identifier (VNI)       |    Reserved   |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                    Variable Length Options                                                               |
   ~                    Variable-Length Options                    ~
   |                                                               |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

Inner Ethernet Header (example payload):
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                 Inner Destination MAC Address                 |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   | Inner Destination MAC Address |   Inner Source MAC Address    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                   Inner Source MAC Address                    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |Optional Ethertype=C-Tag 802.1Q|  Inner VLAN Tag Information   |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

Payload:
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   | Ethertype of Original Payload |                               |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+                               |
   |                                  Original Ethernet Payload    |
   |                                                               |
   |
   ~ (Note that the original Ethernet Frame's Preamble, Start Frame| frame's preamble, start      ~
   |  Delimiter(SFD) & Frame Check Sequence(FCS) frame delimiter (SFD), and frame check sequence (FCS) are not included |
   | included, and the Ethernet Payload payload need not be 4-byte aligned)         | aligned)|
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

Frame Check Sequence:
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |   New Frame Check Sequence (FCS) for Outer Ethernet Frame     |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
]]></artwork>
      </figure>
      </section>
      <section title="Geneve anchor="sec-3.2" numbered="true" toc="default">
        <name>Geneve Packet Format Over IPv6" anchor="section-3.2">

	<figure><artwork><![CDATA[ over IPv6</name>
<figure><name>Geneve Packet Format over IPv6</name>
        <artwork name="" type="" align="left" alt=""><![CDATA[
    0                   1                   2                   3
    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1

Outer Ethernet Header:
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                 Outer Destination MAC Address                 |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   | Outer Destination MAC Address |   Outer Source MAC Address    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                   Outer Source MAC Address                    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |Optional Ethertype=C-Tag 802.1Q|  Outer VLAN Tag Information   |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |       Ethertype=0x86DD    Ethertype = 0x86DD IPv6    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

Outer IPv6 Header:
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |Version| Traffic Class |           Flow Label                  |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |         Payload Length        | NxtHdr=17 UDP |   Hop Limit   |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                                                               |
   +                                                               +
   |                                                               |
   +                     Outer Source IPv6 Address                 +
   |                                                               |
   +                                                               +
   |                                                               |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                                                               |
   +                                                               +
   |                                                               |
   +                  Outer Destination IPv6 Address               +
   |                                                               |
   +                                                               +
   |                                                               |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

Outer UDP Header:
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |       Source Port = xxxx      |    Dest Port = 6081 Geneve    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |           UDP Length          |        UDP Checksum           |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

Geneve Header:
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |Ver|  Opt Len  |O|C|    Rsvd.  |          Protocol Type        |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |        Virtual Network Identifier (VNI)       |    Reserved   |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                    Variable Length Options                                                               |
   ~                    Variable-Length Options                    ~
   |                                                               |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

Inner Ethernet Header (example payload):
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                 Inner Destination MAC Address                 |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   | Inner Destination MAC Address |   Inner Source MAC Address    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                   Inner Source MAC Address                    |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |Optional Ethertype=C-Tag 802.1Q|  Inner VLAN Tag Information   |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

Payload:
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   | Ethertype of Original Payload |                               |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+                               |
   |                                  Original Ethernet Payload    |
   |                                                               |
   |
   ~ (Note that the original Ethernet Frame's Preamble, Start Frame| frame's preamble, start      ~
   |  Delimiter(SFD) & Frame Check Sequence(FCS) frame delimiter (SFD), and frame check sequence (FCS) are not included |
   | included, and the Ethernet Payload payload need not be 4-byte aligned)         | aligned)|
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

Frame Check Sequence:
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |   New Frame Check Sequence (FCS) for Outer Ethernet Frame     |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
]]></artwork>
      </figure>
      </section>
      <section title="UDP Header" anchor="section-3.3"><t> anchor="sec-3.3" numbered="true" toc="default">
        <name>UDP Header</name>
        <t>
   The use of an encapsulating UDP <xref target="RFC0768"/> target="RFC0768" format="default"/> header follows the
   connectionless semantics of Ethernet and IP in addition to providing
   entropy to routers performing ECMP.  The  Therefore, header fields are therefore
   interpreted as follows:</t>

	<t><list style="hanging" hangIndent="3"><t hangText="Source port:">
        <dl newline="false" spacing="normal" indent="3">
          <dt>Source Port:</dt>
          <dd>
            <t>
	A source port selected by the originating tunnel
	<vspace blankLines="0"/> endpoint.  This source port SHOULD <bcp14>SHOULD</bcp14> be the same for all packets
      belonging to a single encapsulated flow to prevent reordering due
      to the use of different paths.  To encourage an even distribution
      of flows across multiple links, the source port SHOULD <bcp14>SHOULD</bcp14> be
      calculated using a hash of the encapsulated packet headers using,
      for example, a traditional 5-tuple.  Since the port represents a
      flow identifier rather than a true UDP connection, the entire
      16-bit range MAY <bcp14>MAY</bcp14> be used to maximize entropy. In addition to setting the source port,
      for IPv6, the flow label MAY <bcp14>MAY</bcp14> also be used for providing entropy. For an example of
      using the IPv6 flow label for tunnel use cases, see <xref target="RFC6438"/>.
	<vspace blankLines="1"/> target="RFC6438" format="default"/>.
            </t>
            <t>
	If Geneve traffic is shared with other UDP listeners
	on the same IP address, tunnel endpoints SHOULD <bcp14>SHOULD</bcp14> implement a mechanism
	to ensure ICMP return traffic arising from network errors is directed
	to the correct listener. The definition of such a mechanism is beyond
	the scope of this document.
            </t>

	<t hangText="Dest port:">
          </dd>
          <dt>Dest Port:</dt>
          <dd>
            <t>
	IANA has assigned port 6081 as the fixed well-known
	<vspace blankLines="0"/> destination port
	for Geneve.  Although the well-known value should be used by default, it is RECOMMENDED <bcp14>RECOMMENDED</bcp14> that implementations make
      this configurable.  The chosen port is used for identification of
      Geneve packets and MUST NOT <bcp14>MUST NOT</bcp14> be reversed for different ends of a
      connection as is done with TCP. It is the responsibility of the control plane for to manage any reconfiguration of the assigned port and its interpretation by respective devices.
      The definition of the control plane is beyond the scope of this document.
            </t>

	<t hangText="UDP length:">
          </dd>
          <dt>UDP Length:</dt>
          <dd>
            <t>
	The length of the UDP packet including the UDP header.
	<vspace blankLines="0"/>
	</t>

	<t hangText="UDP checksum:"> header.</t>
          </dd>
          <dt>UDP Checksum:</dt>
          <dd>
            <t>
	In order to protect the Geneve header, options options, and
	<vspace blankLines="0"/> payload from
	potential data corruption, the UDP checksum SHOULD <bcp14>SHOULD</bcp14> be generated as
	specified in <xref target="RFC0768"/> target="RFC0768" format="default"/> and <xref target="RFC1112"/> target="RFC1122" format="default"/> when
	Geneve is encapsulated in IPv4. To protect the IP header, Geneve header,
	options
	options, and payload from potential data corruption, the UDP checksum MUST <bcp14>MUST</bcp14>
	be generated by default as specified in <xref target="RFC0768"/> target="RFC0768" format="default"/>
	and <xref target="RFC8200"/> target="RFC8200" format="default"/> when Geneve
	is encapsulated in IPv6, except for under certain conditions, which are outlined in the next paragraph.
	Upon receiving such packets with a non-zero UDP checksum,
	the receiving tunnel endpoints MUST <bcp14>MUST</bcp14> validate the checksum.
	If the checksum is not correct, the packet MUST <bcp14>MUST</bcp14> be dropped, otherwise dropped; otherwise,
	the packet MUST <bcp14>MUST</bcp14> be accepted for decapsulation.
	<vspace blankLines="1"/>
            </t>
            <t>
	Under certain conditions, the UDP checksum MAY <bcp14>MAY</bcp14> be set to zero on transmit
	for packets encapsulated in both IPv4 and IPv6 <xref target="RFC8200"/>. target="RFC8200" format="default"/>.
	See <xref target="section-4.3"/> target="sec-4.3" format="default"/> for additional
	requirements that apply when using zero
	UDP checksum with IPv4 and IPv6. Disabling the use of UDP checksums is
	an operational consideration that should take into account the risks
       	and effects of packet corruption.
            </t>

	</list>
	</t>
          </dd>
        </dl>
      </section>
      <section title="Tunnel anchor="sec-3.4" numbered="true" toc="default">
        <name>Tunnel Header Fields" anchor="section-3.4"><t><list style="hanging" hangIndent="3"><t hangText="Ver Fields</name>
        <dl newline="false" spacing="normal" indent="3">
          <dt>Ver (2 bits):"> bits):</dt>
          <dd>
            <t>
	The current version number is 0.  Packets received by
	<vspace blankLines="0"/> a tunnel endpoint with an unknown version MUST <bcp14>MUST</bcp14> be dropped. Transit
      devices interpreting Geneve packets with an unknown
      version number MUST <bcp14>MUST</bcp14> treat them as UDP packets with an unknown
      payload.
            </t>

	<t hangText="Opt
          </dd>
          <dt>Opt Len (6 bits):"> bits):</dt>
          <dd>
            <t>
	The length of the options option fields, expressed in
	<vspace blankLines="0"/>
	four byte 4-byte multiples, not including the eight byte 8-byte fixed tunnel
      header.  This results in a minimum total Geneve header size of 8
      bytes and a maximum of 260 bytes.  The start of the payload
      headers can be found using this offset from the end of the base
      Geneve header.
	<vspace blankLines="1"/>
            </t>
            <t>
   Transit devices MUST <bcp14>MUST</bcp14> maintain consistent forwarding behavior
   irrespective of the value of 'Opt Len', including ECMP link
   selection.
            </t>

	<t hangText="O
          </dd>
          <dt>O (1 bit):"> bit):</dt>
          <dd>
            <t>
	Control packet.  This packet contains a control message.
	<vspace blankLines="0"/> Control messages are sent between tunnel endpoints.
      Tunnel endpoints MUST NOT <bcp14>MUST NOT</bcp14> forward the payload payload,
      and transit devices MUST NOT <bcp14>MUST NOT</bcp14> attempt to interpret it.
      Since control messages are less frequent, it is RECOMMENDED <bcp14>RECOMMENDED</bcp14>
      that tunnel endpoints direct these packets to a high priority high-priority control
      queue (for example, to direct the packet to a general purpose CPU
      from a forwarding ASIC Application-Specific Integrated Circuit (ASIC) or to separate out control traffic on a
      NIC).  Transit devices MUST NOT <bcp14>MUST NOT</bcp14> alter forwarding behavior on the
      basis of this bit, such as ECMP link selection.
            </t>

	<t hangText="C
          </dd>
          <dt>C (1 bit):"> bit):</dt>
          <dd>
            <t>
	Critical options present.  One or more options has the
	<vspace blankLines="0"/> critical bit set (see <xref target="section-3.5"/>). target="sec-3.5" format="default"/>).  If this bit is set set, then
      tunnel endpoints MUST <bcp14>MUST</bcp14> parse the options list to interpret any
      critical options.  On tunnel endpoints where option parsing is not
      supported
      supported, the packet MUST <bcp14>MUST</bcp14> be dropped on the basis of the 'C' bit
      in the base header.  If the bit is not set set, tunnel endpoints MAY <bcp14>MAY</bcp14>
      strip all options using 'Opt Len' and forward the decapsulated
      packet.  Transit devices MUST NOT <bcp14>MUST NOT</bcp14> drop packets on the
      basis of this bit.
            </t>

	<t hangText="Rsvd.
          </dd>
          <dt>Rsvd. (6 bits):"> bits):</dt>
          <dd>
            <t>
	Reserved field, which MUST <bcp14>MUST</bcp14> be zero on transmission
	<vspace blankLines="0"/> and MUST <bcp14>MUST</bcp14> be ignored on receipt.
            </t>

	<t hangText="Protocol
          </dd>
          <dt>Protocol Type (16 bits):"> bits):</dt>
          <dd>
            <t>
	The type of the protocol data unit
	<vspace blankLines="0"/> appearing after the Geneve header.  This follows the EtherType Ethertype
      <xref target="ETYPES"/> convention; target="ETYPES" format="default"/> convention, with Ethernet itself being represented by the
      value 0x6558.
            </t>

	<t hangText="Virtual
          </dd>
          <dt>Virtual Network Identifier (VNI) (24 bits):"> bits):</dt>
          <dd>
            <t>
	An identifier for a
	<vspace blankLines="0"/> unique element of a virtual network.  In many situations situations, this may
      represent an L2 segment, segment; however, the control plane defines the
      forwarding semantics of decapsulated packets.  The VNI MAY <bcp14>MAY</bcp14> be used
      as part of ECMP forwarding decisions or MAY <bcp14>MAY</bcp14> be used as a mechanism
      to distinguish between overlapping address spaces contained in the
      encapsulated packet when load balancing across CPUs.
            </t>

	<t hangText="Reserved
          </dd>
          <dt>Reserved (8 bits):"> bits):</dt>
          <dd>
            <t>
	Reserved field field, which MUST <bcp14>MUST</bcp14> be zero on transmission
	<vspace blankLines="0"/> and ignored on receipt.
            </t>

	</list>
	</t>
          </dd>
        </dl>
      </section>
      <section title="Tunnel Options" anchor="section-3.5"><figure><artwork><![CDATA[ anchor="sec-3.5" numbered="true" toc="default">
        <name>Tunnel Options</name>
      <figure anchor="geneve-options">
        <name>Geneve Option</name>
        <artwork name="" type="" align="left" alt=""><![CDATA[
   0                   1                   2                   3
   0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |          Option Class         |      Type     |R|R|R| Length  |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
   |                      Variable Option Data                                                               |
   ~                  Variable-Length Option Data                  ~
   |                                                               |
   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

                            Geneve Option
]]></artwork>
      </figure>
        <t>
   The base Geneve header is followed by zero or more options in Type-
   Length-Value Type-Length-Value format.  Each option consists of a four byte 4-byte option
   header and a variable amount of option data interpreted according to
   the type.</t>

	<t><list style="hanging" hangIndent="3"><t hangText="Option
        <dl newline="false" spacing="normal" indent="3">
          <dt>Option Class (16 bits):"> bits):</dt>
          <dd>
            <t>
	Namespace for the 'Type' field.  IANA will
	<vspace blankLines="0"/>
	be requested to create has created a "Geneve Option Class" registry to
      allocate identifiers for organizations, technologies, and vendors
      that have an interest in creating types for options.  Each
      organization may allocate types independently to allow
      experimentation and rapid innovation.  It is expected that that, over
      time
      time, certain options will become well known known, and a given
      implementation may use option types from a variety of sources.  In
      addition, IANA will be requested to reserve has reserved specific ranges for
      allocation by IETF Review and for Experimental Use (see <xref target="section-7"/>). target="sec-7" format="default"/>).
            </t>

	<t hangText="Type
          </dd>
          <dt>Type (8 bits):"> bits):</dt>
          <dd>
            <t>
	Type indicating the format of the data contained in
	<vspace blankLines="0"/> this option.  Options are primarily designed to encourage future
      extensibility and innovation innovation, and so standardized forms of these
      options will be defined in separate documents.
	<vspace blankLines="1"/>
            </t>
            <t>
	The high order high-order bit of the option type indicates that this is a
      critical option.  If the receiving tunnel endpoint does not recognize
      this
      the option and this bit is set set, then the packet MUST <bcp14>MUST</bcp14> be dropped.
      If this bit is set in any option option, then the 'C' bit in the
      Geneve base header MUST <bcp14>MUST</bcp14> also be set.  Transit devices MUST NOT <bcp14>MUST NOT</bcp14>
      drop packets on the basis of this bit.  The following figure shows
      the location of the 'C' bit in the 'Type' field:
            </t>

	</list>
	</t>

	<figure><artwork><![CDATA[
          </dd>
        </dl>
<figure><name>&apos;C&apos; Bit in the &apos;Type&apos; Field</name>
        <artwork name="" type="" align="left" alt=""><![CDATA[
   0 1 2 3 4 5 6 7 8
   +-+-+-+-+-+-+-+-+
   |C|    Type     |
   +-+-+-+-+-+-+-+-+
]]></artwork>
</figure>
	<t><list hangIndent="3" style="hanging"><t>
        <dl newline="false" spacing="normal" indent="3">
          <dt/>
          <dd>
      The requirement to drop a packet with an unknown option with the 'C' bit set
      applies to the entire tunnel endpoint system and not a particular
      component of the implementation.  For example, in a system
      comprised of a forwarding ASIC and a general purpose CPU, this
      does not mean that the packet must be dropped in the ASIC.  An
      implementation may send the packet to the CPU using a rate-limited
      control channel for slow-path exception handling.</t>

	</list>
	</t>

	<t><list style="hanging" hangIndent="3"><t hangText="R handling.</dd>
        </dl>
        <dl newline="false" spacing="normal" indent="3">
          <dt>R (3 bits):"> bits):</dt>
          <dd>
	Option control flags reserved for future use.  These bits MUST <bcp14>MUST</bcp14> be
	zero on transmission and MUST <bcp14>MUST</bcp14> be ignored on receipt.
	</t>

	<t hangText="Length
	</dd>
          <dt>Length (5 bits):"> bits):</dt>
          <dd>
            <t>
	Length of the option, expressed in four byte
	<vspace blankLines="0"/>
	multiples 4-byte
	multiples, excluding the option header.  The total length of each
	option may be between 4 and 128 bytes. A value of 0 in the Length 'Length' field implies
       	an option with only an option header and no variable option data. Packets in which the total
      length of all options is not equal to the 'Opt Len' in the base
      header are invalid and MUST <bcp14>MUST</bcp14> be silently dropped if received by a
      tunnel endpoint that processes the options.
            </t>

	<t hangText="Variable
          </dd>
          <dt>Variable-Length Option Data:"> Data:</dt>
          <dd>
            <t>
	Option data interpreted according to 'Type'.
	<vspace blankLines="0"/>
	</t>

	</list>
            </t>

          </dd>
        </dl>
        <section title="Options Processing" anchor="section-3.5.1"><t> anchor="sec-3.5.1" numbered="true" toc="default">
          <name>Options Processing</name>
          <t>
   Geneve options are intended to be originated and processed
   by tunnel endpoints.  However, options MAY <bcp14>MAY</bcp14> be interpreted by transit
   devices along the tunnel path.  Transit devices not
   interpreting Geneve headers (which may or may not include options) MUST <bcp14>MUST</bcp14> handle
   Geneve packets as any other UDP packet and maintain consistent forwarding behavior.</t>
          <t>
   In tunnel endpoints, the generation and interpretation of options is
   determined by the control plane, which is beyond the the scope of this
   document.  However, to ensure interoperability between heterogeneous
   devices
   devices, some requirements are imposed on options and the devices that
   process them:</t>

	<t><list style="symbols"><t>Receiving
          <ul spacing="normal">
            <li>Receiving tunnel endpoints MUST <bcp14>MUST</bcp14> drop packets containing unknown options
      with the 'C' bit set in the option type.  Conversely, transit
      devices MUST NOT <bcp14>MUST NOT</bcp14> drop packets as a result of encountering unknown
      options, including those with the 'C' bit set.</t>

	<t>The set.</li>
            <li>The contents of the options and their ordering MUST NOT <bcp14>MUST NOT</bcp14> be
      modified by transit devices.</t>

	<t>If devices.</li>
            <li>If a tunnel endpoint receives a Geneve packet with an 'Opt Len' (total (the total length of all options)
	that exceeds the options processing options-processing capability of the tunnel endpoint endpoint, then
	the tunnel endpoint MUST <bcp14>MUST</bcp14> drop such packets. An implementation may raise an
	exception to the control plane of in such an event. It is the responsibility
	of the control plane to ensure the communicating peer tunnel endpoints
	have the processing capability to handle the total length of options.
	The definition of the control plane is beyond the scope of this document.</t>
	</list>
	</t> document.</li>
          </ul>
          <t>
   When designing a Geneve option, it is important to consider how the
   option will evolve in the future.  Once an option is defined defined, it is
   reasonable to expect that implementations may come to depend on a
   specific behavior.  As a result, the scope of any future changes must
   be carefully described upfront.</t>
          <t>
   Architecturally, options are intended to be self-descriptive self descriptive and independent.
   This enables parallelism in option options processing and reduces implementation complexity.
   However, the control plane may impose certain ordering restrictions restrictions, as
   described in <xref target="section-4.5.1"/>.</t> target="sec-4.5.1" format="default"/>.</t>
          <t>
   Unexpectedly significant interoperability issues may result from
   changing the length of an option that was defined to be a certain
   size.  A particular option is specified to have either a fixed
   length, which is constant, or a variable length, which may change
   over time or for different use cases.  This property is part of the
   definition of the option and is conveyed by the 'Type'.  For fixed
   length fixed-length options, some implementations may choose to ignore the length 'Length'
   field in the option header and instead parse based on the well known well-known
   length associated with the type.  In this case, redefining the length
   will impact not only the parsing of the option in question but also any
   options that follow.  Therefore, options that are defined to be a fixed
   length in size MUST NOT <bcp14>MUST NOT</bcp14> be redefined to a different length.  Instead,
   a new 'Type' should be allocated. Actual definition of the option type is beyond
   the scope of this document.  The option type and its interpretation should be
   defined by the entity that owns the option class.</t>
          <t>
   Options may be processed by NIC hardware utilizing offloads (e.g. (e.g., LSO and LRO)
   as described in <xref target="section-4.6"/>. target="sec-4.6" format="default"/>. Careful consideration should be
   given to how the offload capabilities outlined in <xref target="section-4.6"/> target="sec-4.6" format="default"/>
   impact an option's design.
          </t>
        </section>
      </section>
    </section>
    <section title="Implementation anchor="sec-4" numbered="true" toc="default">
      <name>Implementation and Deployment Considerations" anchor="section-4"> Considerations</name>
      <section title="Applicability Statement" anchor="section-4.1"><t> anchor="sec-4.1" numbered="true" toc="default">
        <name>Applicability Statement</name>

        <t>
	Geneve is a UDP-based network virtualization overlay encapsulation protocol
	designed to establish tunnels between NVEs over an existing IP network.
	It is intended for use in public or private data center environments,
	for deploying multi-tenant overlay networks over an existing IP underlay network.</t>
        <t>
	Geneve is a UDP based encapsulation protocol transported over existing
	IPv4 and IPv6 networks. Hence, as
	As a UDP based UDP-based protocol, Geneve adheres
	to the UDP usage guidelines as specified in <xref target="RFC8085"/>. target="RFC8085" format="default"/>.
	The applicability of these guidelines are is dependent on the underlay
	IP network and the nature of the Geneve payload protocol
	(example
	(for example, TCP/IP, IP/Ethernet).</t>
        <t>
	Geneve is intended to be deployed in a data center network environment
	operated by a single operator or an adjacent set of cooperating network
	operators that fits with the definition of controlled environments
       	in <xref target="RFC8085"/>. target="RFC8085" format="default"/>. A network in a controlled environment can be
	managed to operate under certain conditions conditions, whereas in the general
   	Internet
   	Internet, this cannot be done.  Hence  Hence, requirements for a tunnel tunneling
   	protocol operating under a controlled environment can be less
   	restrictive than the requirements of the general Internet.
        </t>
        <t>
	For the purpose of this document, a traffic-managed controlled environment
	(TMCE) is defined as an IP network that is traffic-engineered traffic engineered and/or otherwise
	managed (e.g., via use of traffic rate limiters) to avoid congestion. The concept
	of a TMCE is outlined in <xref target="RFC8086"/>. target="RFC8086" format="default"/>. Significant portions of the text
	in <xref target="section-4.1"/> target="sec-4.1" format="default"/> through <xref target="section-4.3"/> target="sec-4.3" format="default"/> are based
	on <xref target="RFC8086"/> target="RFC8086" format="default"/> as applicable to Geneve.</t>
        <t>
	It is the responsibility of the operator to ensure that the guidelines/requirements
	in this section are followed as applicable to their Geneve deployment(s).</t>
      </section>
      <section title="Congestion Control Functionality" anchor="section-4.2"><t> anchor="sec-4.2" numbered="true" toc="default">
        <name>Congestion-Control Functionality</name>
        <t>
	Geneve does not natively provide congestion control congestion-control functionality and relies
	on the payload protocol traffic for congestion control. As such such, Geneve MUST <bcp14>MUST</bcp14>
	be used with congestion controlled congestion-controlled traffic or within a network that is
	traffic managed TMCE to avoid congestion (TMCE). congestion. An operator of a traffic
	managed network (TMCE) TMCE may avoid congestion by through careful provisioning
	of their networks, rate-limiting of user data traffic traffic, and managing traffic
	engineering according to path capacity.</t>
      </section>
      <section title="UDP Checksum" anchor="section-4.3"><t>
	In order anchor="sec-4.3" numbered="true" toc="default">
        <name>UDP Checksum</name>
        <t>
	The outer UDP checksum <bcp14>SHOULD</bcp14> be used with Geneve when transported
   over IPv4; this is to provide integrity of for the Geneve headers, options
   options, and payload, payload in case of data corruption (for example example, to
   avoid misdelivery of the payload to different tenant systems)
	in case of data corruption, the outer UDP checksum SHOULD be used with Geneve
	when transported over IPv4. systems).  The UDP checksum provides a statistical guarantee
	that a payload was not corrupted in transit. These integrity checks are not
	strong from a coding or cryptographic perspective and are not designed to
	detect physical-layer errors or malicious modification of the datagram
	(see Section 3.4 of <xref target="RFC8085"/>). target="RFC8085" sectionFormat="of" section="3.4"/>). In deployments where such a risk exists,
	an operator SHOULD <bcp14>SHOULD</bcp14> use additional data integrity
	mechanisms such as those offered
	by IPsec (see <xref target="section-6.2"/>).</t> target="sec-6.2" format="default"/>).</t>
        <t>
	An operator MAY <bcp14>MAY</bcp14> choose to disable UDP checksums
	and use zero checksums UDP checksum if Geneve packet integrity is provided by other data
	integrity mechanisms mechanisms, such as IPsec or additional checksums checksums, or if one of
	the conditions (a, b, or c) in <xref target="section-4.3.1"/> a, b, c are target="sec-4.3.1" format="default"/> is met.</t>
        <t>
	By default, UDP checksums MUST <bcp14>MUST</bcp14> be used when Geneve is transported over IPv6.
	A tunnel endpoint MAY <bcp14>MAY</bcp14> be configured for use with zero UDP checksum if
	additional requirements in <xref target="section-4.3.1"/> target="sec-4.3.1" format="default"/> are met.</t>
        <section title="UDP Zero anchor="sec-4.3.1" numbered="true" toc="default">
          <name>Zero UDP Checksum Handling with IPv6" anchor="section-4.3.1"><t> IPv6</name>
          <t>
	When Geneve is used over IPv6, the UDP checksum is used to protect IPv6 headers,
	UDP headers headers, and Geneve headers, options options, and payload from potential data corruption.
	As such such, by default default, Geneve MUST <bcp14>MUST</bcp14> use UDP checksums when transported over IPv6.
	An operator MAY <bcp14>MAY</bcp14> choose to configure to operate with zero UDP checksum if
	operating in a traffic managed controlled environment TMCE as stated in
	<xref target="section-4.1"/> target="sec-4.1" format="default"/> if one of the following conditions are is met.</t>

	<t><list style="letters"><t>It
          <ol spacing="normal" type="a">
            <li>It is known that the packet corruption is exceptionally
	unlikely (perhaps based on knowledge of equipment types in their underlay
	network) and the operator is willing to take a risk of undetected packet corruption</t>

	<t>It
	    corruption.</li>
            <li>It is judged through observational measurements (perhaps through historic
	or current traffic flows that use non zero non-zero checksum) that the level of packet
	corruption is tolerably low and is where the operator is willing to take
	the risk of undetected corruption.</t>

	<t>Geneve corruption.</li>
            <li>The Geneve payload is carrying applications that are tolerant of misdelivered
	or corrupted packets (perhaps through higher layer higher-layer checksum validation
	and/or reliability through retransmission) </t>
	</list>
	</t> retransmission). </li>
          </ol>
          <t> In addition addition, Geneve tunnel implementations using zero UDP checksum MUST <bcp14>MUST</bcp14> meet
		the following requirements:</t>

    	<t><list style="numbers"><t>Use
          <ol spacing="normal" type="1">
            <li>Use of UDP checksum over IPv6 MUST <bcp14>MUST</bcp14> be the default
	configuration for all Geneve tunnels.</t>

	<t>If tunnels.</li>
            <li>If Geneve is used with zero UDP checksum over IPv6 IPv6, then such
	    a tunnel
	endpoint implementation MUST <bcp14>MUST</bcp14> meet all the requirements specified
	in Section 4 of <xref target="RFC6936"/> target="RFC6936" sectionFormat="of" section="4"/> and requirement 1 as specified in
	Section 5 of <xref target="RFC6936"/> as that target="RFC6936" sectionFormat="of" section="5"/> since it is relevant to Geneve.</t>

	<t>The Geneve.</li>
            <li>The Geneve tunnel endpoint that decapsulates the tunnel SHOULD
	    <bcp14>SHOULD</bcp14> check that the
	source and destination IPv6 addresses are valid for the Geneve tunnel that
	is configured to receive zero UDP checksum and discard other packets
	for which such a check fails.</t> fails.</li>
            <li>
              <t>The Geneve tunnel endpoint that encapsulates the tunnel MAY <bcp14>MAY</bcp14> use different
	IPv6 source addresses for each Geneve tunnel that uses zero UDP checksum mode
	in order to strengthen the decapsulator's check of the IPv6 source address
	(i.e
	(i.e., the same IPv6 source address is not to be used with more than one IPv6
	destination address, irrespective of whether that destination address is
	a unicast or multicast address). When this is not possible, it is RECOMMENDED <bcp14>RECOMMENDED</bcp14>
	to use each source address for as few Geneve tunnels that use zero UDP
	checksum as is feasible.
	<vspace blankLines="1"/>
              </t>
              <t>
	Note that (for for requirements 3 and 4) 4, the receiving tunnel endpoint can apply
	these checks only if it has out-of-band knowledge that the encapsulating tunnel
	endpoint is applying the indicated behavior. One possibility to obtain this out-of-band
	knowledge is through signaling by the control plane. The definition of
	the control plane is beyond the scope of this document.</t>

	<t>Measures SHOULD
            </li>
            <li>Measures <bcp14>SHOULD</bcp14> be taken to prevent Geneve traffic over IPv6 with zero UDP
	checksum from escaping into the general Internet. Examples of such measures include
	employing packet filters at the gateways or edge of the Geneve network and/or
	keeping logical or physical separation of the Geneve network from networks
	carrying the general Internet traffic.</t>
	</list>
	</t> traffic.</li>
          </ol>
          <t> The above requirements do not change either the requirements
	specified in either <xref target="RFC8200"/> target="RFC8200" format="default"/> or
	the requirements specified in
	<xref target="RFC6936"/>. target="RFC6936" format="default"/>.
          </t>
          <t>The use of the source IPv6 address in addition to the
	destination IPv6 address, plus the recommendation against
 	reuse of source IPv6 addresses among Geneve tunnels tunnels, collectively
	provide some mitigation for the absence of UDP checksum coverage of
	the IPv6 header. A traffic-managed controlled environment that satisfies
       	at least one of the three conditions listed at the beginning of
	this section provides additional assurance.
          </t>

	<t> Editorial Note (The following paragraph to be removed by the
		RFC Editor before publication) </t>
	<t> It was discussed during TSVART early review if the level of requirement for using
		different IPv6 source addresses for different tunnel destinations would need to be "MAY"
		or "SHOULD". The discussion concluded that it was appropriate to keep this
		as "MAY", since it was considered not realistic for control planes having to
		maintain a high level of state on a per tunnel destination basis. In addition, the
		text above provides sufficient guidance to operators and implementors on possible mitigations.</t>
        </section>
      </section>
      <section title="Encapsulation anchor="sec-4.4" numbered="true" toc="default">
        <name>Encapsulation of Geneve in IP" anchor="section-4.4"><t> IP</name>
        <t>
   As an IP-based tunnel tunneling protocol, Geneve shares many properties and
   techniques with existing protocols.  The application of some of these
   are described in further detail, although although, in general general, most concepts
   applicable to the IP layer or to IP tunnels generally also function
   in the context of Geneve.</t>
        <section title="IP Fragmentation" anchor="section-4.4.1"> anchor="sec-4.4.1" numbered="true" toc="default">
          <name>IP Fragmentation</name>
          <t>
   It is strongly RECOMMENDED <bcp14>RECOMMENDED</bcp14> that Path MTU Discovery (<xref target="RFC1191"/>, (see <xref
   target="RFC1191" format="default"/> and <xref target="RFC8201"/>) target="RFC8201" format="default"/>) be used to prevent or minimize fragmentation.
   The use of Path MTU Discovery on the transit network provides the
   encapsulating tunnel endpoint with soft-state information about the link that it may use
   to prevent or minimize fragmentation depending on its role in the
   virtualized network. The NVE can maintain this state (the MTU size of
   the tunnel link(s) associated with the tunnel endpoint), so if a
   tenant system sends large packets that that, when encapsulated encapsulated, exceed the
   MTU size of the tunnel link, the tunnel endpoint can discard such
   packets and send exception messages to the tenant system(s). If the
   tunnel endpoint is associated with a routing or forwarding function and/or has the capability
   to send ICMP messages, the encapsulating tunnel endpoint MAY <bcp14>MAY</bcp14> send ICMP fragmentation
   needed <xref target="RFC0792"/> target="RFC0792" format="default"/> or Packet Too Big <xref target="RFC4443"/> target="RFC4443" format="default"/> messages to the tenant system(s).
   When determining the MTU size of a tunnel link, the maximum length of options MUST <bcp14>MUST</bcp14> be assumed as options may vary
   on a per-packet basis. For example, recommendations/guidance Recommendations and guidance for handling fragmentation in
   similar overlay encapsulation services like PWE3 Pseudowire Emulation
	  Edge-to-Edge (PWE3) are provided in
   Section 5.3 of <xref target="RFC3985"/>.</t> target="RFC3985"
	  sectionFormat="of" section="5.3"/>.</t>
          <t>
   Note that some implementations may not be capable of supporting
   fragmentation or other less common features of the IP header, such as
   options and extension headers. For example, some Some of the issues associated
   with MTU size and fragmentation in IP tunneling and use of ICMP messages is are
   outlined in Section 4.2 of <xref target="I-D.ietf-intarea-tunnels"/>.</t> target="I-D.ietf-intarea-tunnels"
   sectionFormat="of" section="4.2"/>.</t>
        </section>
        <section title="DSCP, ECN anchor="sec-4.4.2" numbered="true" toc="default">
          <name>DSCP, ECN, and TTL" anchor="section-4.4.2"><t> TTL</name>
          <t>
   When encapsulating IP (including over Ethernet) packets in Geneve,
   there are several considerations for propagating DSCP and ECN Differentiated Services
   Code Point (DSCP) and Explicit Congestion Notification (ECN) bits
   from the inner header to the tunnel on transmission and the reverse
   on reception.</t>

          <t>
   <xref target="RFC2983"/> target="RFC2983" format="default"/> provides guidance for mapping DSCP between inner and outer
   IP headers.  Network virtualization is typically more closely aligned
   with the Pipe model described, where the DSCP value on the tunnel
   header is set based on a policy (which may be a fixed value, one
   based on the inner traffic class, class or some other mechanism for
   grouping traffic).  Aspects of the Uniform model (which treats the
   inner and outer DSCP value values as a single field by copying on ingress
   and egress) may also apply, such as the ability to remark re-mark the inner
   header on tunnel egress based on transit marking.  However, the
   Uniform model is not conceptually consistent with network
   virtualization, which seeks to provide strong isolation between
   encapsulated traffic and the physical network.</t>
          <t>
   <xref target="RFC6040"/> target="RFC6040" format="default"/> describes the mechanism for exposing ECN capabilities on IP
   tunnels and propagating congestion markers to the inner packets.
   This behavior MUST <bcp14>MUST</bcp14> be followed for IP packets encapsulated in Geneve.</t>
          <t>
   Though either the Uniform or Pipe models could be used for handling TTL (or Hop Limit in case of IPv6)
   handling when tunneling IP packets, the Pipe model is more aligned consistent with network virtualization.
   <xref target="RFC2003"/> target="RFC2003" format="default"/> provides guidance on handling TTL between inner IP header and outer IP tunnels;
   this model is more aligned with similar to the Pipe model and is RECOMMENDED <bcp14>RECOMMENDED</bcp14> for
   use with Geneve for network virtualization applications.</t>
        </section>
        <section title="Broadcast anchor="sec-4.4.3" numbered="true" toc="default">
          <name>Broadcast and Multicast" anchor="section-4.4.3"><t> Multicast</name>
          <t>
   Geneve tunnels may either be point-to-point unicast between two
   tunnel endpoints or may utilize broadcast or multicast addressing.  It is
   not required that inner and outer addressing match in this respect.
   For example, in physical networks that do not support multicast,
   encapsulated multicast traffic may be replicated into multiple
   unicast tunnels or forwarded by policy to a unicast location
   (possibly to be replicated there).</t>
          <t>
   With physical networks that do support multicast multicast, it may be desirable
   to use this capability to take advantage of hardware replication for
   encapsulated packets.  In this case, multicast addresses may be
   allocated in the physical network corresponding to tenants,
   encapsulated multicast groups, or some other factor.  The allocation
   of these groups is a component of the control plane and therefore and, therefore,
   is beyond the scope of this document.</t>
          <t>
   When physical multicast is in
   use, devices with heterogeneous capabilities may be present in the same group.
   Some options may only be interpretable by a subset of the devices in the group.
   Other devices can safely ignore such options unless the 'C' bit is set to
   mark the unknown option as critical.  Requirements  The requirements outlined in <xref target="section-3.4"/> target="sec-3.4" format="default"/>
   apply for critical options.</t>
          <t>
   In addition, <xref target="RFC8293"/> target="RFC8293" format="default"/> provides examples of various mechanisms that can
   be used for multicast handling in network virtualization overlay networks.</t>
        </section>
        <section title="Unidirectional Tunnels" anchor="section-4.4.4"><t> anchor="sec-4.4.4" numbered="true" toc="default">
          <name>Unidirectional Tunnels</name>
          <t>
   Generally speaking, a Geneve tunnel is a unidirectional concept.  IP
   is not a connection oriented protocol connection-oriented protocol, and it is possible for two
   tunnel endpoints to communicate with each other using different paths or to
   have one side not transmit anything at all.  As Geneve is an IP-based
   protocol, the tunnel layer inherits these same characteristics.</t>
          <t>
   It is possible for a tunnel to encapsulate a protocol, such as TCP,
   which
   that is connection oriented and maintains session state at that
   layer.  In addition, implementations MAY <bcp14>MAY</bcp14> model Geneve tunnels as
   connected, bidirectional links, such as for example, to provide the abstraction of
   a virtual port.  In both of these cases, bidirectionality of the
   tunnel is handled at a higher layer and does not affect the operation
   of Geneve itself.</t>
        </section>
      </section>
      <section title="Constraints anchor="sec-4.5" numbered="true" toc="default">
        <name>Constraints on Protocol Features" anchor="section-4.5"><t> Features</name>
        <t>
   Geneve is intended to be flexible to for use with a wide range of current and
   future applications.  As a result, certain constraints may be placed
   on the use of metadata or other aspects of the protocol in order to
   optimize for a particular use case.  For example, some applications
   may limit the types of options which that are supported or enforce a
   maximum number or length of options.  Other applications may only
   handle certain encapsulated payload types, such as Ethernet or IP.
   This could
   These optimizations can be implemented either globally throughout (throughout
   the system or, for system) or locally (for example, restricted to certain classes
   of devices or network paths.</t> paths).</t>
        <t>
   These constraints may be communicated to tunnel endpoints either
   explicitly through a control plane or implicitly by the nature of the
   application.  As Geneve is defined as a data plane protocol that is
   control plane agnostic, definition of such mechanisms are is beyond the scope of this
   document.</t>
        <section title="Constraints anchor="sec-4.5.1" numbered="true" toc="default">
          <name>Constraints on Options" anchor="section-4.5.1"><t> Options</name>
          <t>
   While Geneve options are flexible, a control plane may restrict
   the number of option TLVs as well as the order and size of the TLVs
   between tunnel endpoints to make it simpler for a data plane
   implementation in software or hardware to handle (see <xref target="I-D.ietf-nvo3-encap"/>. target="I-D.ietf-nvo3-encap" format="default"/>).
   For example, there may be some critical information information, such as a secure
   hash
   hash, that must be processed in a certain order to provide the lowest
   latency
   latency, or there may be other scenarios where the options must be
	  processed in a certain given order due to protocol semantics.</t>
          <t>
   A control plane may negotiate a subset of option TLVs and certain TLV
   ordering, as well
   ordering; it may also limit the total number of option TLVs present
   in the packet, for example, to accommodate hardware capable of
   processing fewer options <xref target="I-D.ietf-nvo3-encap"/>. options.  Hence, a control plane
   needs to have the ability to describe the supported TLVs TLV subset and
   their order
   its ordering to the tunnel endpoints.  In the absence of a control
   plane, alternative configuration mechanisms may be used for this
   purpose.  Such mechanisms are beyond the scope of this document.</t>
        </section>
      </section>
      <section title="NIC Offloads" anchor="section-4.6"><t> anchor="sec-4.6" numbered="true" toc="default">
        <name>NIC Offloads</name>

        <t>
   Modern NICs currently provide a variety of offloads to enable the
   efficient processing of packets.  The implementation of many of these
   offloads requires only that the encapsulated packet be easily parsed
   (for example, checksum offload).  However, optimizations such as LSO
   and LRO involve some processing of the options themselves since they
   must be replicated/merged across multiple packets.  In these
   situations, it is desirable to not to require changes to the offload
   logic to handle the introduction of new options.  To enable this,
   some constraints are placed on the definitions of options to allow
   for simple processing rules:</t>

	<t><list style="symbols"><t>When
        <ul spacing="normal">
          <li>When performing LSO, a NIC MUST <bcp14>MUST</bcp14> replicate the entire Geneve header
      and all options, including those unknown to the device, onto each
      resulting segment unless an option allows an exception.
      Conversely, when performing LRO, a NIC may assume that a
      binary comparison of the options (including unknown options) is
      sufficient to ensure equality and MAY <bcp14>MAY</bcp14> merge packets with equal
      Geneve headers.</t>

	<t>Options MUST NOT headers.</li>
          <li>Options <bcp14>MUST NOT</bcp14> be reordered during the course of offload
      processing, including when merging packets for the purpose of LRO.</t>

	<t>NICs LRO.</li>
          <li>NICs performing offloads MUST NOT <bcp14>MUST NOT</bcp14> drop packets with unknown
      options, including those marked as critical, unless explicitly configured.</t>

	</list>
	</t> configured to do so.</li>
        </ul>
        <t>
   There is no requirement that a given implementation of Geneve employ
   the offloads listed as examples above.  However, as these offloads
   are currently widely deployed in commercially available NICs, the
   rules described here are intended to enable efficient handling of
   current and future options across a variety of devices.</t>
      </section>
      <section title="Inner anchor="sec-4.7" numbered="true" toc="default">
        <name>Inner VLAN Handling" anchor="section-4.7"><t> Handling</name>
        <t>
   Geneve is capable of encapsulating a wide range of protocols and
   therefore protocols; therefore, a given implementation is likely to support only a small
   subset of the possibilities.  However, as Ethernet is expected to be
   widely deployed, it is useful to describe the behavior of VLANs
   inside encapsulated Ethernet frames.</t>
        <t>
   As with any protocol, support for inner VLAN headers is OPTIONAL. <bcp14>OPTIONAL</bcp14>.  In
   many cases, the use of encapsulated VLANs may be disallowed due to
   security or implementation considerations.  However, in other cases cases, the trunking of VLAN frames across a Geneve tunnel can prove useful.  As
   a result, the processing of inner VLAN tags upon ingress or egress
   from a tunnel endpoint is based upon the configuration of the tunnel
   endpoint and/or control plane and is not explicitly defined as part of
   the data format.</t>
      </section>
    </section>
    <section title="Transition Considerations" anchor="section-5"><t> anchor="sec-5" numbered="true" toc="default">
      <name>Transition Considerations</name>
      <t>
   Viewed exclusively from the data plane, Geneve is compatible with existing IP networks
   as it appears to most devices as UDP packets.
   However, as there are already a number of tunnel tunneling protocols deployed
   in network virtualization environments, there is a practical question
   of transition and coexistence.</t>
      <t>
   Since Geneve builds on the base data plane functionality provided by the most
   common protocols used for network virtualization (VXLAN, NVGRE) (VXLAN and NVGRE),
   it should be straightforward to port an existing control plane
   to run on top of it with minimal effort.  With both the old and new
   packet formats supporting the same set of capabilities, there is no
   need for a hard transition - transition; tunnel endpoints directly communicating with
   each other can use any common protocol, which may be different even
   within a single overall system.

   As transit devices are primarily
   forwarding packets on the basis of the IP header, all protocols
   appear similar to be similar, and these devices do not introduce additional
   interoperability concerns.</t>
      <t>
   To assist with this transition, it is strongly suggested that
   implementations support simultaneous operation of both Geneve and
   existing tunnel protocols tunneling protocols, as it is expected to be common for a single
   node to communicate with a mixture of other nodes. Eventually, older
   protocols may be phased out as they are no longer in use.</t>
    </section>
    <section title="Security Considerations" anchor="section-6"><t> anchor="sec-6" numbered="true" toc="default">
      <name>Security Considerations</name>

      <t>
   As it is encapsulated within a UDP/IP packet, Geneve does not have any inherent security
   mechanisms.
   As a result, an attacker with access to the underlay
   network transporting the IP packets has the ability to snoop, alter snoop on, alter, or
   inject packets.  Compromised tunnel endpoints or transit devices may also
   spoof identifiers in the tunnel header to gain access to networks
   owned by other tenants.</t>
      <t>
   Within a particular security domain, such as a data center operated
   by a single service provider, the most common and highest performing highest-performing security
   mechanism is isolation of trusted components.  Tunnel traffic can be
   carried over a separate VLAN and filtered at any untrusted
   boundaries.</t>
      <t>
   When crossing an untrusted link, such as the general Internet, VPN technologies such as IPsec
   <xref target="RFC4301"/> target="RFC4301" format="default"/> should be used to provide authentication and/or encryption of
   the IP packets formed as part of Geneve encapsulation (See (see <xref target="section-6.1.1"/>).</t> target="sec-6.1.1" format="default"/>).</t>
      <t>
   Geneve does not otherwise affect the security of the encapsulated
   packets. As per the guidelines of BCP 72 <xref target="RFC3552"/>, target="RFC3552" format="default"/>, the following sections
   describe potential security risks that may be applicable to Geneve deployments
   and approaches to mitigate such risks. It is also noted that not all such risks are applicable
   to all Geneve deployment scenarios, i.e., only a subset may be applicable to certain deployments.
   So an
   An operator has to make an assessment based on their network environment and
   environment, determine the risks that are applicable to their specific environment environment, and use appropriate mitigation approaches as applicable. </t>
      <section title="Data Confidentiality" anchor="section-6.1"><t> anchor="sec-6.1" numbered="true" toc="default">
        <name>Data Confidentiality</name>
        <t>
	Geneve is a network virtualization overlay encapsulation protocol
	designed to establish tunnels between NVEs
	over an existing IP network. It can be used to deploy multi-tenant overlay networks
       	over an existing IP underlay network in a public or private data center.

	The overlay service is typically provided by a service provider, for example such as a
	cloud services service provider or a private data center operator, this operator. This may or not may be
	the same provider as an underlay service provider. Due to the nature of multi-tenancy in such environments,
	a tenant system may expect data confidentiality to ensure its packet data is not tampered with
	(active
	(i.e., active attack) in transit or is a target of unauthorized
	monitoring (passive attack) (i.e., passive attack), for example example, by other tenant systems or underlay service provider.
	A compromised network node or a transit device within a
	data center may passively monitor Geneve packet data between NVEs; NVEs or route
	traffic for further inspection. A tenant may
	expect the overlay service provider to provide data confidentiality as part of the service service, or
	a tenant may bring its own data confidentiality mechanisms like IPsec or TLS to protect the data
	end to end between its tenant systems. The overlay provider is expected to provide
   	cryptographic protection in cases where the underlay provider is not the
   	same as the overlay provider to ensure the payload is not exposed to the underlay.</t>
        <t>
	If an operator determines data confidentiality is necessary in their environment
	based on their risk analysis, analysis -- for example as example, in multi-tenant environments,
	environments -- then an encryption mechanism SHOULD <bcp14>SHOULD</bcp14> be used to encrypt the tenant
	data end to end between the NVEs. The NVEs may use existing well established well-established
	encryption mechanisms mechanisms, such as IPsec, DTLS, etc.</t>
        <section title="Inter-Data anchor="sec-6.1.1" numbered="true" toc="default">

          <name>Inter-Data Center Traffic" anchor="section-6.1.1"><t> Traffic</name>
          <t>
	A tenant system in a customer premises (private data center) may want to connect
	to tenant systems on their tenant overlay network in a public cloud data center center, or a tenant may want to have its tenant systems located in multiple geographically
	separated data centers for high availability. Geneve data traffic between tenant systems
	across such separated networks should be protected from threats when traversing public networks.
	Any Geneve overlay data leaving the data center network beyond the operator's security domain
	SHOULD
	<bcp14>SHOULD</bcp14> be secured by encryption mechanisms mechanisms, such as
	IPsec or other VPN technologies technologies, to protect the communications between the NVEs
	when they are geographically separated over untrusted network links. Specification of
	data protection mechanisms employed between data centers is beyond the scope of this document.</t>
          <t>
	The principles described in <xref target="section-4"/> target="sec-4" format="default"/> regarding controlled environments still apply to
	the geographically separated data center usage outlined in this section.</t>
        </section>
      </section>
      <section title="Data Integrity" anchor="section-6.2"><t> anchor="sec-6.2" numbered="true" toc="default">
        <name>Data Integrity</name>
        <t>
	Geneve encapsulation is used between NVEs to establish overlay tunnels over an existing
	IP underlay network. In a multi-tenant data center,  a rogue or compromised tenant system
	may try to launch a passive attack attack, such as monitoring the traffic of other tenants, or an
	active attack attack, such as trying to inject unauthorized Geneve encapsulated traffic such
	as spoofing, replay, etc., into the network. To prevent such attacks, an NVE MUST NOT <bcp14>MUST NOT</bcp14>
	propagate Geneve packets beyond the NVE to tenant systems and SHOULD <bcp14>SHOULD</bcp14> employ packet filtering packet-filtering
	mechanisms so as not to forward unauthorized traffic between tenant systems in different tenant networks.
	An NVE MUST NOT <bcp14>MUST NOT</bcp14> interpret Geneve packets from tenant systems other than as frames to be encapsulated.</t>
        <t>
	A compromised network node or a transit device within a data center may launch an active
	attack trying to tamper with the Geneve packet data between NVEs. Malicious tampering of
	Geneve header fields may cause the packet from one tenant to be forwarded to a different
	tenant network. If an operator determines the there is a possibility of such a threat in their environment,
	the operator may choose to employ data integrity mechanisms between NVEs. In order to prevent
	such risks, a data integrity mechanism SHOULD <bcp14>SHOULD</bcp14> be used in such environments to protect the
	integrity of Geneve packets packets, including packet headers, options options, and payload on communications
	between NVE pairs. A cryptographic data protection mechanism mechanism, such as IPsec IPsec, may be used to
	provide data integrity protection. A data center operator may choose to deploy any other
	data integrity mechanisms as applicable and supported in their underlay networks,
	although non-cryptographic mechanisms may not protect the Geneve portion of the packet from tampering. </t>
      </section>
      <section title="Authentication anchor="sec-6.3" numbered="true" toc="default">
        <name>Authentication of NVE peers" anchor="section-6.3"><t> Peers</name>
        <t>
	A rogue network device or a compromised NVE in a data center environment might be able to
	spoof Geneve packets as if it came from a legitimate NVE. In order to mitigate such a risk,
	an operator SHOULD <bcp14>SHOULD</bcp14> use an authentication mechanism, such as IPsec IPsec, to ensure that the
	Geneve packet originated from the intended NVE peer, peer in environments where the operator
	determines spoofing or rogue devices is a are potential threat. threats. Other simpler source checks checks,
	such as ingress filtering for VLAN/MAC/IP address, addresses, reverse path forwarding checks, etc.,
	may be used in certain trusted environments to ensure Geneve packets originated
       	from the intended NVE peer.</t>
      </section>
      <section title="Options anchor="sec-6.4" numbered="true" toc="default">
        <name>Options Interpretation by Transit Devices" anchor="section-6.4"><t> Devices</name>
        <t>
	Options, if present in the packet, are generated and terminated by tunnel endpoints. As indicated
	in <xref target="section-2.2.1"/>, target="sec-2.2.1" format="default"/>, transit devices may interpret the options. However,
	if the packet is protected by encryption from tunnel endpoint
	to tunnel endpoint encryption, for example (for example, through IPsec, IPsec), transit devices will not have visibility into the Geneve header or options
	in the packet.  In such cases cases, transit devices MUST <bcp14>MUST</bcp14> handle Geneve packets as any other IP packet
	and maintain consistent forwarding behavior. In cases where options are interpreted by transit devices, the operator
	MUST
	<bcp14>MUST</bcp14> ensure that transit devices are trusted and not compromised. The definition of
	a mechanism to ensure this trust is beyond the scope of this document.</t>
      </section>
      <section title="Multicast/Broadcast" anchor="section-6.5"><t> anchor="sec-6.5" numbered="true" toc="default">
        <name>Multicast/Broadcast</name>
        <t>
	In typical data center networks where IP multicasting is not supported in the underlay
	network, multicasting may be supported using multiple unicast tunnels. The same security
	requirements as described in the above sections can be used to protect Geneve communications
	between NVE peers. If IP multicasting is supported in the underlay network and the operator
	chooses to use it for multicast traffic among tunnel endpoints, then the operator in such
	environments may use data protection mechanisms mechanisms, such as IPsec with multicast
	extensions <xref target="RFC5374"/> target="RFC5374" format="default"/>, to protect multicast traffic among Geneve NVE groups.</t>
      </section>
      <section title="Control anchor="sec-6.6" numbered="true" toc="default">
        <name>Control Plane Communications" anchor="section-6.6"><t> Communications</name>
        <t>
	A Network Virtualization Authority (NVA) as outlined in <xref target="RFC8014"/> target="RFC8014" format="default"/> may
	be used as a control plane for configuring and managing the Geneve NVEs. The data center
	operator is expected to use security mechanisms to protect the communications between
	the NVA to and NVEs and to use authentication mechanisms to detect any rogue or compromised
	NVEs within their administrative domain.  Data protection mechanisms for control plane
	communication or authentication mechanisms between the NVA and the NVEs are beyond
	the scope of this document.</t>
      </section>
    </section>
    <section title="IANA Considerations" anchor="section-7"><t> anchor="sec-7" numbered="true" toc="default">
      <name>IANA Considerations</name>
      <t>
	IANA has allocated UDP port 6081 in the Service "Service Name and Transport Protocol
	Port Number Registry Registry" <xref target="IANA-SN"/> target="IANA-SN" format="default"/> as the well-known destination port
       	for Geneve based on early registration.</t>

	<t>Upon publication of this document, this registration will have its reference changed to cite
   	this document [RFC-to-be] and inline with <xref target="RFC6335"/> the assignee and contact of the port entry should be
   	changed to IESG &lt;iesg@ietf.org&gt; and IETF Geneve:</t>
<dl newline="false" spacing="compact">
<dt>Service Name:</dt><dd>geneve</dd>
<dt>Transport Protocol(s):</dt><dd>UDP</dd>
<dt>Assignee:</dt><dd>IESG &lt;iesg@ietf.org&gt;</dd>
<dt>Contact:</dt><dd>IETF Chair &lt;chair@ietf.org&gt; respectively:</t>

<figure><artwork><![CDATA[
Service Name: geneve
Transport Protocol(s): UDP
Assignee: IESG <iesg@ietf.org>
Contact: IETF Chair <chair@ietf.org>
Description: Generic &lt;chair@ietf.org&gt;</dd>
<dt>Description:</dt><dd>Generic Network Virtualization Encapsulation (Geneve)
Reference: [RFC-to-be]
Port Number: 6081
]]></artwork>
</figure> (Geneve)</dd>
<dt>Reference:</dt><dd>[RFC8926]</dd>
<dt>Port Number:</dt><dd>6081</dd>
</dl>
      <t>
   In addition, IANA is requested to create has created a new subregistry titled "Geneve Option Class"
   registry to allocate Option Classes.
   for option classes. This registry is to be has been placed under
   a new Network "Network Virtualization Overlay (NVO3) protocols page (to be created) (NVO3)" heading in the IANA protocol registries <xref target="IANA-PR"/>. target="IANA-PR" format="default"/>.
   The Geneve "Geneve Option Class Class" registry shall consist consists of
   16-bit hexadecimal values along with descriptive strings, assignee/contact information information, and references.
   The registration rules for the new registry are (as defined by <xref target="RFC8126"/>):</t>

	<texttable style="full"><ttcol> Range</ttcol>
	<ttcol> target="RFC8126" format="default"/>):</t>
      <table align="center"> <name>Geneve Option Class Registry Ranges</name>
        <thead>
          <tr>
            <th align="left"> Range</th>
            <th align="left"> Registration Procedures</ttcol>
	<c>0x0000..0x00FF</c>
	<c>IETF Review</c>
	<c>0x0100..0xFEFF</c>
	<c>First Procedures</th>
          </tr>
        </thead>
        <tbody>
          <tr>
            <td align="left">0x0000-0x00FF</td>
            <td align="left">IETF Review</td>
          </tr>
          <tr>
            <td align="left">0x0100-0xFEFF</td>
            <td align="left">First Come First Served</c>
	<c>0xFF00..0xFFFF</c>
	<c>Experimental Use</c>
	</texttable>

	<t>
   Initial registrations in the new registry are as follows:</t>

	<texttable style="full"><ttcol> Option Class</ttcol>
		<ttcol> Description</ttcol> <ttcol> Assignee/Contact </ttcol> <ttcol> References</ttcol>
	<c>0x0100</c>
	<c>Linux</c>
	<c></c>
	<c></c>
	<c>0x0101</c>
	<c>Open vSwitch (OVS)</c>
	<c></c>
	<c></c>
	<c>0x0102</c>
	<c>Open Virtual Networking (OVN)</c>
	<c></c>
	<c></c>
	<c>0x0103</c>
	<c>In-band Network Telemetry (INT)</c>
	<c></c>
	<c></c>
	<c>0x0104</c>
	<c>VMware, Inc.</c>
	<c></c>
	<c></c>
	<c>0x0105</c>
	<c>Amazon.com, Inc.</c>
	<c></c>
	<c></c>
	<c>0x0106</c>
	<c>Cisco Systems, Inc.</c>
	<c></c>
	<c></c>
	<c>0x0107</c>
	<c>Oracle Corporation</c>
	<c></c>
	<c></c>
	<c>0x0108..0x0110</c>
	<c>Amazon.com, Inc.</c>
	<c></c>
	<c></c>
	</texttable>
	</section>

	<section title="Contributors" anchor="section-8"><t>
   The following individuals were authors of an earlier version of this
   document and made significant contributions:</t>

	<figure><artwork><![CDATA[
Pankaj Garg
Microsoft Corporation
1 Microsoft Way
Redmond, WA  98052
USA

Email: pankajg@microsoft.com

Chris Wright
Red Hat Inc.
1801 Varsity Drive
Raleigh, NC  27606
USA

Email: chrisw@redhat.com

Kenneth Duda
Arista Networks
5453 Great America Parkway
Santa Clara, CA  95054
USA

Email: kduda@arista.com

Dinesh G. Dutt
Independent

Email: didutt@gmail.com

Jon Hudson
Independent

Email: jon.hudson@gmail.com

Ariel Hendel
Facebook, Inc.
1 Hacker Way
Menlo Park, CA  94025
USA

Email: ahendel@fb.com
]]></artwork>
	</figure>
	</section>

	<section title="Acknowledgements" anchor="section-9">
	<t>
	The authors wish to acknowledge Puneet Agarwal, David Black, Sami Boutros, Scott Bradner,
	Martin Casado, Alissa Cooper, Roman Danyliw, Bruce Davie, Anoop Ghanwani, Benjamin Kaduk,
	Suresh Krishnan, Mirja Kuhlewind, Barry Leiba, Daniel Migault, Greg Mirksy, Tal Mizrahi,
	Kathleen Moriarty, Magnus Nystrom, Adam Roach, Sabrina Tanamal, Dave Thaler, Eric Vyncke,
	Magnus Westerlund and many other members of the NVO3 WG for their reviews, comments and suggestions.</t>

	<t>
	The authors would like to thank Sam Aldrin, Alia Atlas, Matthew Bocci, Benson Schliesser, and Martin Vigoureux
	for their guidance throughout the process.</t> Served</td>
          </tr>
          <tr>
            <td align="left">0xFF00-0xFFFF</td>
            <td align="left">Experimental Use</td>
          </tr>
        </tbody>
      </table>
    </section>

  </middle>
  <back>
	<references title="Normative References">
	&RFC0768;
	&RFC0792;
	&RFC1112;
	&RFC1191;
	&RFC2003;
	&RFC2119;
	&RFC4443;
	&RFC6040;
	&RFC6936;
	&RFC7365;
	&RFC8085;
	&RFC8126;
	&RFC8174;
	&RFC8200;
	&RFC8201;

<displayreference target="I-D.ietf-nvo3-encap" to="NVO3-ENCAP"/>
<displayreference target="I-D.ietf-nvo3-dataplane-requirements" to="NVO3-DATAPLANE"/>
<displayreference target="I-D.ietf-intarea-tunnels" to="INTAREA-TUNNELS"/>

    <references>
      <name>References</name>
      <references>
        <name>Normative References</name>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.0768.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.0792.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.1122.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.1191.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.2003.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.2119.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.4443.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.6040.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.6936.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.7365.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8085.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8126.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8174.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8200.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8201.xml"/>
      </references>
	<references title="Informative References">
      <references>
        <name>Informative References</name>

        <reference anchor="ETYPES" target="https://www.iana.org/assignments/ieee-802-numbers">
          <front>
            <title>IEEE 802 Numbers</title>
            <author>
	      <organization>The IEEE Registration Authority</organization>
              <organization>IANA</organization>
            </author>
	    <date/>
          </front>
        </reference>

	&I-D.ietf-nvo3-encap;
	&I-D.ietf-nvo3-dataplane-requirements;
	&I-D.ietf-intarea-tunnels;

        <xi:include href="https://datatracker.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-nvo3-encap.xml"/>

        <xi:include href="https://datatracker.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-nvo3-dataplane-requirements.xml"/>

        <xi:include href="https://datatracker.ietf.org/public/rfc/bibxml3/reference.I-D.ietf-intarea-tunnels.xml"/>

        <reference anchor="IANA-PR" target="https://www.iana.org/protocols">
          <front>
            <title>Protocol Registries</title>
            <author>
              <organization>IANA</organization>
            </author>
	    <date/>
               </front>
        </reference>

        <reference anchor="IANA-SN" target="https://www.iana.org/assignments/service-names-port-numbers">
          <front>
            <title>Service Name and Transport Protocol Port Number Registry</title>
            <author>
              <organization>IANA</organization>
            </author>
	    <date/>
              </front>
        </reference>

	<!--&IEEE.802.1Q_2014;-->

<reference anchor='IEEE.802.1Q_2018' target='http://ieeexplore.ieee.org/servlet/opac?punumber=8403925'> anchor="IEEE.802.1Q_2018" target="http://ieeexplore.ieee.org/servlet/opac?punumber=8403925">
          <front>
            <title>IEEE Standard for Local and Metropolitan Area Networks--Bridges and Bridged Networks</title>
            <seriesInfo name="DOI" value="10.1109/IEEESTD.2018.8403927"/>
            <seriesInfo name="IEEE" value="802.1Q-2018"/>
            <author>
              <organization>IEEE</organization>
            </author>
            <date day='06' month='July' year='2018' />
  <abstract><t>This standard specifies how the Media Access Control (MAC) Service is supported by Bridged Networks,
	the principles of operation of those networks, and the operation of MAC Bridges and VLAN Bridges,
	including management, protocols, and algorithms</t>
   </abstract> month="July" year="2018"/>
          </front>
 <seriesInfo name='IEEE' value='802.1Q-2018' />
 <seriesInfo name='DOI' value='10.1109/ieeestd.2018.8403927' />
        </reference>

	&RFC2983;
	&RFC3031;
	&RFC3552;
    	&RFC3985;
	&RFC4301;
	&RFC5374;
	&RFC6335;
	&RFC6438;
	&RFC7348;
	&RFC7637;
	&RFC8014;
	&RFC8086;
	&RFC8293;
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.2983.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.3031.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.3552.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.3985.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.4301.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.5374.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.6438.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.7348.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.7637.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8014.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8086.xml"/>
        <xi:include href="https://xml2rfc.ietf.org/public/rfc/bibxml/reference.RFC.8293.xml"/>

        <reference anchor="VL2"
        target="https://www.sigcomm.org/sites/default/files/ccr/papers/2009/October/1594977-1592576.pdf"> target="https://dl.acm.org/doi/10.1145/1594977.1592576">
          <front>
            <title>VL2: A Scalable and Flexible Data Center Network</title>
            <seriesInfo name="DOI" value="10.1145/1594977.1592576"/>
            <author surname="Greenberg, A., et al.">
                <organization></organization>
              <organization/>
            </author>
            <date year="2009" /> month="August" year="2009"/>
          </front>
        <seriesInfo name="ACM SIGCOMM" value="Computer
       <refcontent>ACM SIGCOMM Computer Communication
        Review"/>
        <seriesInfo name="DOI" value="10.1145/1594977.1592576"/> Review</refcontent>
        </reference>
      </references>
    </references>
   <section anchor="sec-9" numbered="false" toc="default">
      <name>Acknowledgements</name>
      <t>
	The authors wish to acknowledge <contact fullname="Puneet Agarwal"/>,
	<contact fullname="David Black"/>, <contact fullname="Sami Boutros"/>,
	  <contact fullname="Scott Bradner"/>,
	<contact fullname="Martín Casado"/>, <contact fullname="Alissa Cooper"/>,
	  <contact fullname="Roman Danyliw"/>, <contact fullname="Bruce Davie"/>,
	    <contact fullname="Anoop Ghanwani"/>, <contact fullname="Benjamin
	      Kaduk"/>, <contact fullname="Suresh Krishnan"/>, <contact
	      fullname="Mirja Kühlewind"/>, <contact fullname="Barry Leiba"/>,
	      <contact fullname="Daniel Migault"/>, <contact fullname="Greg
		Mirksy"/>, <contact fullname="Tal Mizrahi"/>,
	<contact fullname="Kathleen Moriarty"/>, <contact fullname="Magnus
	  Nyström"/>, <contact fullname="Adam Roach"/>, <contact fullname="Sabrina
	  Tanamal"/>, <contact fullname="Dave Thaler"/>, <contact fullname="Éric Vyncke"/>,
	<contact fullname="Magnus Westerlund"/>, and many other members of the NVO3 Working Group for their reviews, comments, and suggestions.</t>
      <t>
	The authors would like to thank <contact fullname="Sam Aldrin"/>,
	<contact fullname="Alia Atlas"/>, <contact fullname="Matthew Bocci"/>,
	  <contact fullname="Benson Schliesser"/>, and <contact fullname="Martin Vigoureux"/>
	for their guidance throughout the process.</t>
    </section>

   <section anchor="sec-8" numbered="false" toc="default">
      <name>Contributors</name>
      <t>
   The following individuals were authors of an earlier version of this
   document and made significant contributions:</t>

    <contact fullname="Pankaj Garg" >
        <organization>Microsoft Corporation</organization>
        <address>
          <postal>
            <street>1 Microsoft Way</street>
            <city>Redmond</city>
            <region>WA</region><code>98052</code>
            <country>United States of America</country>
          </postal>
          <email>pankajg@microsoft.com</email>
        </address>
    </contact>

    <contact fullname="Chris Wright" >
        <organization>Red Hat Inc.</organization>
        <address>
          <postal>
            <street>1801 Varsity Drive</street>
            <city>Raleigh</city>
            <region>NC</region><code>27606</code>
            <country>United States of America</country>
          </postal>
          <email>chrisw@redhat.com</email>
        </address>
    </contact>

    <contact fullname="Kenneth Duda" >
        <organization>Arista Networks</organization>
        <address>
          <postal>
            <street>5453 Great America Parkway</street>
            <city>Santa Clara</city>
            <region>CA</region><code>95054</code>
            <country>United States of America</country>
          </postal>
          <email>kduda@arista.com</email>
        </address>
    </contact>

    <contact fullname="Dinesh G. Dutt" >
        <organization>Independent</organization>
        <address>
          <postal>
            <street></street>
            <city></city>
            <region></region><code></code>
            <country></country>
          </postal>
          <email>didutt@gmail.com</email>
        </address>
    </contact>

    <contact fullname="Jon Hudson" >
        <organization>Independent</organization>
        <address>
          <postal>
            <street></street>
            <city></city>
            <region></region><code></code>
            <country></country>
          </postal>
          <email>jon.hudson@gmail.com</email>
        </address>
    </contact>

    <contact fullname="Ariel Hendel" >
        <organization>Facebook, Inc.</organization>
        <address>
          <postal>
            <street>1 Hacker Way</street>
            <city>Menlo Park</city>
            <region>CA</region><code>94025</code>
            <country>United States of America</country>
          </postal>
          <email>ahendel@fb.com</email>
        </address>
    </contact>
   </section>

  </back>
</rfc>