feature: create tap link for firecracker compute

cmd/main.go

@@ -2,6 +2,8 @@
 
 package main
 
+// inspired by https://github.com/firecracker-microvm/firectl/blob/main/main.go#L64
+
 import (
 	"antoine-roux.tk/projects/go/firecracker-netns/internal/netlink"
 	"antoine-roux.tk/projects/go/firecracker-netns/internal/netns"
@@ -48,9 +50,28 @@ func setupEnv() int {
 
 	err = netns.Set(newNs)
 	if err != nil {
+		fmt.Println("set guest ns error", err)
 		return 1
 	}
 
+	firstIpTapNetwork := net.IPv4(172, 16, 0, 1)
+	tapNetwork := net.IPNet{
+		IP:   firstIpTapNetwork,
+		Mask: net.CIDRMask(30, 32),
+	}
+
+	tap, err := netlink.CreateTap(tapNetwork, vethPair.Guest.Link.Attrs().Name)
+	if err != nil {
+		fmt.Println("create tap in guest ns error", err)
+		return 1
+	}
+	defer func(tap *netlink.Tap) {
+		err := tap.DeleteTap()
+		if err != nil {
+			fmt.Println("delete tap error", err)
+		}
+	}(tap)
+
 	// Do something with the network namespace
 	interfaces, _ := net.Interfaces()
 	fmt.Printf("Interfaces: %v\n", interfaces)

go.mod

@@ -3,13 +3,11 @@ module antoine-roux.tk/projects/go/firecracker-netns
 go 1.21.5
 
 require (
+	github.com/coreos/go-iptables v0.7.0
 	github.com/docker/docker v24.0.7+incompatible
 	github.com/hashicorp/go-version v1.6.0
 	github.com/vishvananda/netlink v1.2.1-beta.2
 	golang.org/x/sys v0.15.0
 )
 
-require (
+require github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect
-	github.com/coreos/go-iptables v0.7.0 // indirect
-	github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect
-)

go.sum

@@ -6,8 +6,9 @@ github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mO
 github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs=
 github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
-github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA=
+github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=

internal/netlink/tap.go

@@ -0,0 +1,182 @@
+package netlink
+
+import (
+	"antoine-roux.tk/projects/go/firecracker-netns/internal/sysctl"
+	"crypto/rand"
+	"fmt"
+	"github.com/coreos/go-iptables/iptables"
+	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
+	"net"
+	"os/user"
+	"strconv"
+)
+
+// inspired by https://github.com/firecracker-microvm/firecracker-go-sdk/blob/main/cni/internal/netlink.go#L197
+
+type Tap struct {
+	Link             netlink.Link
+	Ip               net.IPNet
+	externalLinkName string
+}
+
+// CreateTap set up a tap device with random number
+// this tap is linked with routing to given externalLinkName
+func CreateTap(tapNetwork net.IPNet, externalLinkName string) (*Tap, error) {
+	tapName, err := randomTapName()
+	if err != nil {
+		return nil, err
+	}
+
+	// Get current user information
+	current, err := user.Current()
+	if err != nil {
+		return nil, err
+	}
+	gid, err := strconv.Atoi(current.Uid)
+	if err != nil {
+		return nil, err
+	}
+	uid, err := strconv.Atoi(current.Gid)
+	if err != nil {
+		return nil, err
+	}
+
+	tapLinkAttrs := netlink.NewLinkAttrs()
+	tapLinkAttrs.Name = tapName
+	tapLink := &netlink.Tuntap{
+		LinkAttrs: tapLinkAttrs,
+
+		// We want a tap device (L2) as opposed to a tun (L3)
+		Mode: netlink.TUNTAP_MODE_TAP,
+
+		// Firecracker does not support multiqueue tap devices at this time:
+		// https://github.com/firecracker-microvm/firecracker/issues/750
+		Queues: 1,
+
+		// single queue tap device
+		// parse vnet headers added by the vm's virtio_net implementation
+		Flags: unix.IFF_TAP | netlink.TUNTAP_ONE_QUEUE | netlink.TUNTAP_VNET_HDR,
+
+		Owner: uint32(uid),
+		Group: uint32(gid),
+	}
+
+	err = netlink.LinkAdd(tapLink)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, tapFd := range tapLink.Fds {
+		err := tapFd.Close()
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Re-fetch the container link to get its creation-time parameters, e.g. index and mac
+	fmt.Println(tapName)
+	tap, err := netlink.LinkByName(tapName)
+	if err != nil {
+		netlink.LinkDel(tapLink)
+		return nil, err
+	}
+
+	// Ip tapAddr add <tap Network> dev tap0
+	err = netlink.AddrAdd(tap, &netlink.Addr{
+		IPNet: &tapNetwork,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	err = netlink.LinkSetUp(tap)
+	if err != nil {
+		return nil, err
+	}
+
+	err = sysctl.Set(fmt.Sprintf("net.ipv4.conf.%s.proxy_arp", tap.Attrs().Name), "1")
+	if err != nil {
+		return nil, err
+	}
+
+	// echo 1 > /proc/sys/net/ipv4/ip_forward
+	err = sysctl.Set("net.ipv4.ip_forward", "1")
+	if err != nil {
+		return nil, err
+	}
+
+	ipt, err := iptables.New()
+	if err != nil {
+		return nil, err
+	}
+
+	// sudo iptables -t nat -I POSTROUTING 1 -o vethee574a50 -j MASQUERADE --source 192.168.0.3/24
+	err = ipt.InsertUnique("nat", "POSTROUTING", 1, "-o", externalLinkName, "-j", "MASQUERADE", "--source", tapNetwork.String())
+	if err != nil {
+		return nil, err
+	}
+
+	// sudo iptables -I FORWARD 1 -o tap0 -i vethee574a50 -j ACCEPT
+	err = ipt.InsertUnique("filter", "FORWARD", 1, "-o", externalLinkName, "-i", tap.Attrs().Name, "-j", "ACCEPT")
+	if err != nil {
+		return nil, err
+	}
+
+	// sudo iptables -I FORWARD 1 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+	err = ipt.InsertUnique("filter", "FORWARD", 1, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT")
+	if err != nil {
+		return nil, err
+	}
+
+	return &Tap{tap, tapNetwork, externalLinkName}, nil
+}
+
+// randomTapName returns string "tap" with random prefix (hashed from entropy)
+func randomTapName() (string, error) {
+	entropy := make([]byte, 4)
+	_, err := rand.Read(entropy)
+	if err != nil {
+		return "", fmt.Errorf("failed to generate random tap name: %v", err)
+	}
+
+	// NetworkManager (recent versions) will ignore veth devices that start with "veth"
+	return fmt.Sprintf("tap%x", entropy), nil
+}
+
+// DeleteTap delete tap device and related routing
+func (tap *Tap) DeleteTap() error {
+	link, err := netlink.LinkByName(tap.Link.Attrs().Name)
+	if err != nil {
+		return err
+	}
+
+	if err := netlink.LinkDel(link); err != nil {
+		return fmt.Errorf("failed to delete %q: %v", link.Attrs().Name, err)
+	}
+
+	ipt, err := iptables.New()
+	if err != nil {
+		return err
+	}
+
+	// sudo iptables -t nat -I POSTROUTING 1 -o vethee574a50 -j MASQUERADE --source 192.168.0.3/24
+	err = ipt.DeleteIfExists("nat", "POSTROUTING", "-o", tap.externalLinkName, "-j", "MASQUERADE", "--source", tap.Ip.String())
+	if err != nil {
+		return err
+	}
+
+	// sudo iptables -I FORWARD 1 -o tap0 -i vethee574a50 -j ACCEPT
+	err = ipt.DeleteIfExists("filter", "FORWARD", "-o", tap.Link.Attrs().Name, "-i", tap.externalLinkName, "-j", "ACCEPT")
+	if err != nil {
+		return err
+	}
+
+	// sudo iptables -I FORWARD 1 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
+	err = ipt.DeleteIfExists("filter", "FORWARD", "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

internal/netlink/veth.go

@@ -13,14 +13,14 @@ import (
 // inspired by https://github.com/containernetworking/plugins/blob/main/pkg/ip/link_linux.go#L57
 
 type Peer struct {
-	link  netlink.Link
+	Link  netlink.Link
-	netNs netns.NsHandle
+	NetNs netns.NsHandle
-	ip    net.IPNet
+	Ip    net.IPNet
 }
 
 type PairLink struct {
-	host        Peer
+	Host        Peer
-	guest       Peer
+	Guest       Peer
 	wanLinkName string
 }
 
@@ -107,11 +107,6 @@ func NewVirtualPairing(ns netns.NsHandle, wanLinkName string) (pair *PairLink, e
 		return nil, err
 	}
 
-	err = netlink.LinkSetUp(veth1)
-	if err != nil {
-		return nil, err
-	}
-
 	veth1IP := &netlink.Addr{
 		IPNet: &net.IPNet{
 			IP:   net.IPv4(192, 168, 0, 4),
@@ -125,6 +120,11 @@ func NewVirtualPairing(ns netns.NsHandle, wanLinkName string) (pair *PairLink, e
 		return nil, err
 	}
 
+	err = netlink.LinkSetUp(veth1)
+	if err != nil {
+		return nil, err
+	}
+
 	// echo 1 > /proc/sys/net/ipv4/conf/veth4bbc9fef/proxy_arp
 	err = sysctl.Set(fmt.Sprintf("net.ipv4.conf.%s.proxy_arp", veth1.Attrs().Name), "1")
 	if err != nil {
@@ -161,21 +161,21 @@ func NewVirtualPairing(ns netns.NsHandle, wanLinkName string) (pair *PairLink, e
 	}
 
 	return &PairLink{
-		host:        Peer{veth1, origins, *veth1IP.IPNet},
+		Host:        Peer{veth1, origins, *veth1IP.IPNet},
-		guest:       Peer{veth2, ns, *veth2IP.IPNet},
+		Guest:       Peer{veth2, ns, *veth2IP.IPNet},
 		wanLinkName: wanLinkName,
 	}, nil
 }
 
-// DeleteVirtualPairing removes an interface link.
+// DeleteVirtualPairing removes an interface Link.
 func (p *PairLink) DeleteVirtualPairing() error {
-	err := netns.Set(p.host.netNs)
+	err := netns.Set(p.Host.NetNs)
 	if err != nil {
-		return fmt.Errorf("set original Ns for deleting veth %q: %v", p.host.link.Attrs().Name, err)
+		return fmt.Errorf("set original Ns for deleting veth %q: %v", p.Host.Link.Attrs().Name, err)
 	}
 
-	if err := netlink.LinkDel(p.host.link); err != nil {
+	if err := netlink.LinkDel(p.Host.Link); err != nil {
-		return fmt.Errorf("failed to delete %q: %v", p.host.link.Attrs().Name, err)
+		return fmt.Errorf("failed to delete %q: %v", p.Host.Link.Attrs().Name, err)
 	}
 
 	ipt, err := iptables.New()
@@ -184,13 +184,13 @@ func (p *PairLink) DeleteVirtualPairing() error {
 	}
 
 	// sudo iptables -t nat -I POSTROUTING 1 -o wlp3s0 -j MASQUERADE --source 192.168.0.3/24
-	err = ipt.DeleteIfExists("nat", "POSTROUTING", "-o", p.wanLinkName, "-j", "MASQUERADE", "--source", p.guest.ip.String())
+	err = ipt.DeleteIfExists("nat", "POSTROUTING", "-o", p.wanLinkName, "-j", "MASQUERADE", "--source", p.Guest.Ip.String())
 	if err != nil {
 		return err
 	}
 
 	// sudo iptables -I FORWARD 1 -o vethee574a50 -i wlp3s0 -j ACCEPT
-	err = ipt.DeleteIfExists("filter", "FORWARD", "-o", p.host.link.Attrs().Name, "-i", p.wanLinkName, "-j", "ACCEPT")
+	err = ipt.DeleteIfExists("filter", "FORWARD", "-o", p.Host.Link.Attrs().Name, "-i", p.wanLinkName, "-j", "ACCEPT")
 	if err != nil {
 		return err
 	}