Support IPv6 addresses in the OPTE zone setup service

illumos-utils/src/ipadm.rs

@@ -29,6 +29,9 @@ const ADDROBJ_ALREADY_EXISTS: &str = "Address object already exists";
 
 pub enum AddrObjType {
     DHCP,
+    // NOTE: This can result in more than one address, if there is a DHCPv6
+    // server on the same link as the addrobj. That happens most often for OPTE
+    // ports used in zones that need external connectivity.
     AddrConf,
     Static(IpAddr),
 }
@@ -83,6 +86,9 @@ impl Ipadm {
     /// Remove any scope from an IPv6 address.
     /// e.g. fe80::8:20ff:fed0:8687%oxControlService1/10 ->
     ///      fe80::8:20ff:fed0:8687/10
+    //
+    // TODO-cleanup: This could directly parse the line into an IpNet if
+    // possible, rather than emitting a new string.
     fn remove_addr_scope(input: &str) -> String {
         if let Some(pos) = input.find('%') {
             let (base, rest) = input.split_at(pos);
@@ -98,6 +104,10 @@ impl Ipadm {
 
     /// Return the IP network associated with an address object, or None if
     /// there is no address object with this name.
+    //
+    // TODO-correctness: There can be many addresses associated with an addrobj,
+    // e.g., for IPv6 where we have link-local + DHCPv6 addresses. This will
+    // ignore all but the first.
     pub async fn addrobj_addr(
         addrobj: &str,
     ) -> Result<Option<IpNet>, ExecutionError> {
@@ -173,6 +183,7 @@ impl Ipadm {
         Ok(())
     }
 
+    /// Create a link-local IPv6 addrconf address and a static IPv6 address.
     pub async fn create_static_and_autoconfigured_addrs(
         datalink: &str,
         listen_addr: &Ipv6Addr,
@@ -192,15 +203,6 @@ impl Ipadm {
         Ok(())
     }
 
-    // Create gateway on the IP interface if it doesn't already exist
-    pub async fn create_opte_gateway(
-        opte_iface: &String,
-    ) -> Result<(), ExecutionError> {
-        let addrobj = format!("{}/public", opte_iface);
-        Self::ensure_ip_addrobj_exists(&addrobj, AddrObjType::DHCP).await?;
-        Ok(())
-    }
-
     /// Set TCP recv_buf to 1 MB.
     pub async fn set_tcp_recv_buf() -> Result<(), ExecutionError> {
         let mut cmd = Command::new(PFEXEC);

illumos-utils/src/opte/mod.rs

@@ -43,6 +43,11 @@ use std::net::Ipv4Addr;
 use std::net::Ipv6Addr;
 
 /// Information about the gateway for an OPTE port
+///
+/// TODO-remove: This only exists to communicate the destination for a default
+/// IPv4 route from the port's private IP to the OPTE "virtual gateway". We can
+/// remove this entirely when we resolve
+/// <https://github.com/oxidecomputer/omicron/issues/2931>.
 #[derive(Debug, Clone, Copy)]
 #[allow(dead_code)]
 pub struct Gateway {

illumos-utils/src/opte/port.rs

@@ -114,6 +114,7 @@ impl Port {
         &self.inner.name
     }
 
+    // TODO-remove: <https://github.com/oxidecomputer/omicron/issues/2931>
     pub fn gateway(&self) -> &Gateway {
         &self.inner.gateway
     }

illumos-utils/src/route.rs

@@ -5,6 +5,7 @@
 //! Utilities for manipulating the routing tables.
 
 use crate::zone::ROUTE;
+use crate::zone::ZLOGIN;
 use crate::{
     ExecutionError, PFEXEC, command_to_string, execute_async,
     output_to_exec_error,
@@ -16,75 +17,117 @@ use omicron_common::address::{
 use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 use tokio::process::Command;
 
-/// Wraps commands for interacting with routing tables.
-pub struct Route {}
+#[derive(Clone, Copy, Debug)]
+pub enum RouteDestination {
+    /// The "default" route, suitable for any gateway.
+    ///
+    /// This is only used in OPTE routing setup.
+    Default,
+    /// An AZ IPv6 /48 subnet. Used only for underlay routing setup.
+    Subnet(Ipv6Subnet<AZ_PREFIX_LENGTH>),
+}
 
-#[derive(Debug, Clone, Copy)]
-pub enum Gateway {
-    Ipv4(Ipv4Addr),
-    Ipv6(Ipv6Addr),
+impl RouteDestination {
+    const fn check_gateway(&self, gateway: IpAddr) -> Result<(), RouteError> {
+        match self {
+            // Works for either
+            RouteDestination::Default => Ok(()),
+            RouteDestination::Subnet(_) => {
+                if gateway.is_ipv6() {
+                    Ok(())
+                } else {
+                    Err(RouteError::IncompatibleGatewayAndDestination)
+                }
+            }
+        }
+    }
 }
 
-impl Route {
-    pub async fn ensure_default_route_with_gateway(
-        gateway: Gateway,
-        datalink: &str,
-    ) -> Result<(), ExecutionError> {
-        Self::ensure_route_with_gateway("default", gateway, datalink).await
+impl core::fmt::Display for RouteDestination {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            RouteDestination::Default => f.write_str("default"),
+            RouteDestination::Subnet(net) => net.fmt(f),
+        }
     }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum RouteError {
+    #[error(
+        "Destination and gateway are incompatible, because they \
+        have different IP versions"
+    )]
+    IncompatibleGatewayAndDestination,
+    #[error(transparent)]
+    Exec(#[from] ExecutionError),
+}
 
+/// Wraps commands for interacting with routing tables.
+pub struct Route {}
+
+impl Route {
+    /// Ensure there is an interface route to the underlay subnet on the
+    /// provided link.
     pub async fn ensure_underlay_route_with_gateway(
         gateway: Ipv6Addr,
         datalink: &str,
-    ) -> Result<(), ExecutionError> {
+    ) -> Result<(), RouteError> {
         // Route to the underlay AZ's /48 by deriving it from the gateway IP.
-        let underlay_az: Ipv6Subnet<AZ_PREFIX_LENGTH> =
-            Ipv6Subnet::new(gateway);
-        let gateway = Gateway::Ipv6(gateway);
+        let underlay_az = Ipv6Subnet::new(gateway);
         Self::ensure_route_with_gateway(
-            &underlay_az.to_string(),
-            gateway,
+            None,
+            RouteDestination::Subnet(underlay_az),
+            IpAddr::V6(gateway),
             datalink,
         )
         .await
     }
 
+    /// Ensure there is an interface route for the `gateway_ip` to
+    /// `destination` on the provided `datalink`.
     async fn ensure_route_with_gateway(
-        destination: &str,
-        gateway: Gateway,
+        zone: Option<&str>,
+        destination: RouteDestination,
+        gateway_ip: IpAddr,
         datalink: &str,
-    ) -> Result<(), ExecutionError> {
+    ) -> Result<(), RouteError> {
+        destination.check_gateway(gateway_ip)?;
+        let destination = destination.to_string();
         let inet;
         let gw;
-        match gateway {
-            Gateway::Ipv4(addr) => {
+        match gateway_ip {
+            IpAddr::V4(addr) => {
                 inet = "-inet";
                 gw = addr.to_string();
             }
-            Gateway::Ipv6(addr) => {
+            IpAddr::V6(addr) => {
                 inet = "-inet6";
                 gw = addr.to_string();
             }
         }
         // Add the desired route if it doesn't already exist
         let mut cmd = Command::new(PFEXEC);
+        if let Some(zone) = zone {
+            cmd.args([ZLOGIN, zone]);
+        }
         let cmd = cmd.args(&[
             ROUTE,
             "-n",
             "get",
             inet,
-            destination,
+            &destination,
             inet,
             &gw,
             "-ifp",
             datalink,
         ]);
 
         let out = cmd.output().await.map_err(|err| {
-            ExecutionError::ExecutionStart {
+            RouteError::Exec(ExecutionError::ExecutionStart {
                 command: command_to_string(cmd.as_std()),
                 err,
-            }
+            })
         })?;
         match out.status.code() {
             Some(0) => (),
@@ -93,42 +136,114 @@ impl Route {
             // When that is the case, we'll add the route.
             Some(ESRCH) => {
                 let mut cmd = Command::new(PFEXEC);
+                if let Some(zone) = zone {
+                    cmd.args([ZLOGIN, zone]);
+                }
                 let cmd = cmd.args(&[
                     ROUTE,
                     "add",
                     inet,
-                    destination,
+                    &destination,
                     inet,
                     &gw,
                     "-ifp",
                     datalink,
                 ]);
-                execute_async(cmd).await?;
+                execute_async(cmd).await.map_err(RouteError::from)?;
             }
             Some(_) | None => {
-                return Err(output_to_exec_error(cmd.as_std(), &out));
+                return Err(RouteError::Exec(output_to_exec_error(
+                    cmd.as_std(),
+                    &out,
+                )));
             }
         };
         Ok(())
     }
 
-    pub async fn ensure_opte_route(
-        gateway: &Ipv4Addr,
-        iface: &String,
-        opte_ip: &IpAddr,
+    /// Configure an IPv4 route to the OPTE virtual gateway.
+    ///
+    /// # Details
+    ///
+    /// OPTE acts as the "virtual gateway" for all traffic from the private IP
+    /// address. By design, we always configure OPTE with a /32 or /128 address,
+    /// which means there are no other addresses "on-link", i.e., whose
+    /// addresses can be resolved through ARP or NDP. OPTE itself, however, is
+    /// on-link, and receives all traffic from the guest.
+    ///
+    /// But before that happens, the illumos kernel looks at an IP packet from
+    /// the guest and has to decide where to route it. If the destination
+    /// address is on-link, then the kernel will send an ARP or NDP request for
+    /// that address, resolve it, and there we go. But like we said above, _no_
+    /// addresses are on-link. So how does the kernel learn any routes?
+    ///
+    /// For IPv6, this all happens automagically through NDP. We can create an
+    /// IPv6 link-local address with just the MAC address, and then send out
+    /// Router Solicitations. OPTE will respond with Router Advertisements,
+    /// advertising itself as a default router. The guest side will
+    /// automatically learn to send all traffic to OPTE's virtual gateway
+    /// address. (The actual address is _also_ learned through NDP. It's great.)
+    ///
+    /// For IPv4, things are harder. The mechanism for learning a default route
+    /// is the DHCP Classless Static Route Option (#121, in RFC 3442). When the
+    /// guest asks for a DHCP server and gets a lease back, that can include
+    /// this information about the virtual gateway and route, similar to NDP.
+    /// Unfortunately, the illumos `dhcpagent` doesn't understand this option.
+    /// Therefore, we need to manually program this information using
+    /// `route(8)`.
+    ///
+    /// This method adds a route to the single-host virtual gateway address
+    /// provided in `gateway_ip`, and then ensures there's also a default route
+    /// that sends all traffic from the guest out to the gateway.
+    ///
+    /// TODO-remove: We should pull all this shenanigans out when we resolve
+    /// https://github.com/oxidecomputer/stlouis/issues/326. Doing so is tracked
+    /// by https://github.com/oxidecomputer/omicron/issues/2931. At that point,
+    /// the only thing we'll need to do is create the DHCP / addrconf `ipadm`
+    /// addrobjs for V4 and / or V6, and then the protocols will do the rest.
+    pub async fn configure_opte_virtual_gateway_ipv4_route(
+        zone: Option<&str>,
+        opte_port: &str,
+        gateway_ip: &Ipv4Addr,
+        private_ip: &Ipv4Addr,
+    ) -> Result<(), RouteError> {
+        Self::ensure_opte_route(zone, opte_port, gateway_ip, private_ip)
+            .await?;
+        Self::ensure_route_with_gateway(
+            zone,
+            RouteDestination::Default,
+            IpAddr::V4(*gateway_ip),
+            opte_port,
+        )
+        .await
+    }
+
+    /// Ensure there is a host route from the private IP to the OPTE virtual
+    /// gateway address, for the provided port.
+    async fn ensure_opte_route(
+        zone: Option<&str>,
+        opte_port: &str,
+        gateway_ip: &Ipv4Addr,
+        private_ip: &Ipv4Addr,
     ) -> Result<(), ExecutionError> {
         // Add the desired route if it doesn't already exist
         let mut cmd = Command::new(PFEXEC);
+        let gateway_ip = gateway_ip.to_string();
+        let private_ip = private_ip.to_string();
+        if let Some(zone) = zone {
+            cmd.args([ZLOGIN, zone]);
+        }
         let cmd = cmd.args(&[
             ROUTE,
             "-n",
             "get",
+            "-inet",
             "-host",
-            &gateway.to_string(),
-            &opte_ip.to_string(),
+            &gateway_ip,
+            &private_ip,
             "-interface",
             "-ifp",
-            &iface.to_string(),
+            opte_port,
         ]);
 
         let out = cmd.output().await.map_err(|err| {
@@ -144,15 +259,19 @@ impl Route {
             // When that is the case, we'll add the route.
             Some(ESRCH) => {
                 let mut cmd = Command::new(PFEXEC);
+                if let Some(zone) = zone {
+                    cmd.args([ZLOGIN, zone]);
+                }
                 let cmd = cmd.args(&[
                     ROUTE,
                     "add",
+                    "-inet",
                     "-host",
-                    &gateway.to_string(),
-                    &opte_ip.to_string(),
+                    &gateway_ip,
+                    &private_ip,
                     "-interface",
                     "-ifp",
-                    &iface.to_string(),
+                    opte_port,
                 ]);
                 execute_async(cmd).await?;
             }
@@ -182,3 +301,21 @@ impl Route {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    const V4_ADDR: Ipv4Addr = Ipv4Addr::new(10, 0, 0, 1);
+    const V4: IpAddr = IpAddr::V4(V4_ADDR);
+    const V6_ADDR: Ipv6Addr = Ipv6Addr::new(0xfd00, 0, 0, 0, 0, 0, 0, 1);
+    const V6: IpAddr = IpAddr::V6(V6_ADDR);
+
+    #[test]
+    fn check_gateway() {
+        let subnet = Ipv6Subnet::new(V6_ADDR);
+        assert!(RouteDestination::Default.check_gateway(V4).is_ok());
+        assert!(RouteDestination::Default.check_gateway(V6).is_ok());
+        assert!(RouteDestination::Subnet(subnet).check_gateway(V4).is_err());
+        assert!(RouteDestination::Subnet(subnet).check_gateway(V6).is_ok());
+    }
+}