From 1f637ccc772763ccbf6fd58c1225bf2f342dd787 Mon Sep 17 00:00:00 2001 From: RouxAntoine Date: Wed, 3 Jan 2024 22:20:12 +0100 Subject: [PATCH] feature: create tap link for firecracker compute --- cmd/main.go | 21 +++ go.mod | 6 +- go.sum | 3 +- internal/netlink/tap.go | 182 +++++++++++++++++++++++ internal/netlink/{netlink.go => veth.go} | 38 ++--- 5 files changed, 226 insertions(+), 24 deletions(-) create mode 100644 internal/netlink/tap.go rename internal/netlink/{netlink.go => veth.go} (89%) diff --git a/cmd/main.go b/cmd/main.go index b49ce72..291148f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,6 +2,8 @@ package main +// inspired by https://github.com/firecracker-microvm/firectl/blob/main/main.go#L64 + import ( "antoine-roux.tk/projects/go/firecracker-netns/internal/netlink" "antoine-roux.tk/projects/go/firecracker-netns/internal/netns" @@ -48,9 +50,28 @@ func setupEnv() int { err = netns.Set(newNs) if err != nil { + fmt.Println("set guest ns error", err) return 1 } + firstIpTapNetwork := net.IPv4(172, 16, 0, 1) + tapNetwork := net.IPNet{ + IP: firstIpTapNetwork, + Mask: net.CIDRMask(30, 32), + } + + tap, err := netlink.CreateTap(tapNetwork, vethPair.Guest.Link.Attrs().Name) + if err != nil { + fmt.Println("create tap in guest ns error", err) + return 1 + } + defer func(tap *netlink.Tap) { + err := tap.DeleteTap() + if err != nil { + fmt.Println("delete tap error", err) + } + }(tap) + // Do something with the network namespace interfaces, _ := net.Interfaces() fmt.Printf("Interfaces: %v\n", interfaces) diff --git a/go.mod b/go.mod index 251d315..770a5a3 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,11 @@ module antoine-roux.tk/projects/go/firecracker-netns go 1.21.5 require ( + github.com/coreos/go-iptables v0.7.0 github.com/docker/docker v24.0.7+incompatible github.com/hashicorp/go-version v1.6.0 github.com/vishvananda/netlink v1.2.1-beta.2 golang.org/x/sys v0.15.0 ) -require ( - github.com/coreos/go-iptables v0.7.0 // indirect - github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae // indirect -) +require github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect diff --git a/go.sum b/go.sum index bdbed21..c6d8882 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,9 @@ github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mO github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f h1:p4VB7kIXpOQvVn1ZaTIVp+3vuYAXFe3OJEvjbUYJLaA= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= diff --git a/internal/netlink/tap.go b/internal/netlink/tap.go new file mode 100644 index 0000000..5b2292a --- /dev/null +++ b/internal/netlink/tap.go @@ -0,0 +1,182 @@ +package netlink + +import ( + "antoine-roux.tk/projects/go/firecracker-netns/internal/sysctl" + "crypto/rand" + "fmt" + "github.com/coreos/go-iptables/iptables" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + "net" + "os/user" + "strconv" +) + +// inspired by https://github.com/firecracker-microvm/firecracker-go-sdk/blob/main/cni/internal/netlink.go#L197 + +type Tap struct { + Link netlink.Link + Ip net.IPNet + externalLinkName string +} + +// CreateTap set up a tap device with random number +// this tap is linked with routing to given externalLinkName +func CreateTap(tapNetwork net.IPNet, externalLinkName string) (*Tap, error) { + tapName, err := randomTapName() + if err != nil { + return nil, err + } + + // Get current user information + current, err := user.Current() + if err != nil { + return nil, err + } + gid, err := strconv.Atoi(current.Uid) + if err != nil { + return nil, err + } + uid, err := strconv.Atoi(current.Gid) + if err != nil { + return nil, err + } + + tapLinkAttrs := netlink.NewLinkAttrs() + tapLinkAttrs.Name = tapName + tapLink := &netlink.Tuntap{ + LinkAttrs: tapLinkAttrs, + + // We want a tap device (L2) as opposed to a tun (L3) + Mode: netlink.TUNTAP_MODE_TAP, + + // Firecracker does not support multiqueue tap devices at this time: + // https://github.com/firecracker-microvm/firecracker/issues/750 + Queues: 1, + + // single queue tap device + // parse vnet headers added by the vm's virtio_net implementation + Flags: unix.IFF_TAP | netlink.TUNTAP_ONE_QUEUE | netlink.TUNTAP_VNET_HDR, + + Owner: uint32(uid), + Group: uint32(gid), + } + + err = netlink.LinkAdd(tapLink) + if err != nil { + return nil, err + } + + for _, tapFd := range tapLink.Fds { + err := tapFd.Close() + if err != nil { + return nil, err + } + } + + // Re-fetch the container link to get its creation-time parameters, e.g. index and mac + fmt.Println(tapName) + tap, err := netlink.LinkByName(tapName) + if err != nil { + netlink.LinkDel(tapLink) + return nil, err + } + + // Ip tapAddr add dev tap0 + err = netlink.AddrAdd(tap, &netlink.Addr{ + IPNet: &tapNetwork, + }) + if err != nil { + return nil, err + } + + err = netlink.LinkSetUp(tap) + if err != nil { + return nil, err + } + + err = sysctl.Set(fmt.Sprintf("net.ipv4.conf.%s.proxy_arp", tap.Attrs().Name), "1") + if err != nil { + return nil, err + } + + // echo 1 > /proc/sys/net/ipv4/ip_forward + err = sysctl.Set("net.ipv4.ip_forward", "1") + if err != nil { + return nil, err + } + + ipt, err := iptables.New() + if err != nil { + return nil, err + } + + // sudo iptables -t nat -I POSTROUTING 1 -o vethee574a50 -j MASQUERADE --source 192.168.0.3/24 + err = ipt.InsertUnique("nat", "POSTROUTING", 1, "-o", externalLinkName, "-j", "MASQUERADE", "--source", tapNetwork.String()) + if err != nil { + return nil, err + } + + // sudo iptables -I FORWARD 1 -o tap0 -i vethee574a50 -j ACCEPT + err = ipt.InsertUnique("filter", "FORWARD", 1, "-o", externalLinkName, "-i", tap.Attrs().Name, "-j", "ACCEPT") + if err != nil { + return nil, err + } + + // sudo iptables -I FORWARD 1 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + err = ipt.InsertUnique("filter", "FORWARD", 1, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT") + if err != nil { + return nil, err + } + + return &Tap{tap, tapNetwork, externalLinkName}, nil +} + +// randomTapName returns string "tap" with random prefix (hashed from entropy) +func randomTapName() (string, error) { + entropy := make([]byte, 4) + _, err := rand.Read(entropy) + if err != nil { + return "", fmt.Errorf("failed to generate random tap name: %v", err) + } + + // NetworkManager (recent versions) will ignore veth devices that start with "veth" + return fmt.Sprintf("tap%x", entropy), nil +} + +// DeleteTap delete tap device and related routing +func (tap *Tap) DeleteTap() error { + link, err := netlink.LinkByName(tap.Link.Attrs().Name) + if err != nil { + return err + } + + if err := netlink.LinkDel(link); err != nil { + return fmt.Errorf("failed to delete %q: %v", link.Attrs().Name, err) + } + + ipt, err := iptables.New() + if err != nil { + return err + } + + // sudo iptables -t nat -I POSTROUTING 1 -o vethee574a50 -j MASQUERADE --source 192.168.0.3/24 + err = ipt.DeleteIfExists("nat", "POSTROUTING", "-o", tap.externalLinkName, "-j", "MASQUERADE", "--source", tap.Ip.String()) + if err != nil { + return err + } + + // sudo iptables -I FORWARD 1 -o tap0 -i vethee574a50 -j ACCEPT + err = ipt.DeleteIfExists("filter", "FORWARD", "-o", tap.Link.Attrs().Name, "-i", tap.externalLinkName, "-j", "ACCEPT") + if err != nil { + return err + } + + // sudo iptables -I FORWARD 1 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + err = ipt.DeleteIfExists("filter", "FORWARD", "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT") + if err != nil { + return err + } + + return nil +} diff --git a/internal/netlink/netlink.go b/internal/netlink/veth.go similarity index 89% rename from internal/netlink/netlink.go rename to internal/netlink/veth.go index 4c85d59..6aec153 100644 --- a/internal/netlink/netlink.go +++ b/internal/netlink/veth.go @@ -13,14 +13,14 @@ import ( // inspired by https://github.com/containernetworking/plugins/blob/main/pkg/ip/link_linux.go#L57 type Peer struct { - link netlink.Link - netNs netns.NsHandle - ip net.IPNet + Link netlink.Link + NetNs netns.NsHandle + Ip net.IPNet } type PairLink struct { - host Peer - guest Peer + Host Peer + Guest Peer wanLinkName string } @@ -107,11 +107,6 @@ func NewVirtualPairing(ns netns.NsHandle, wanLinkName string) (pair *PairLink, e return nil, err } - err = netlink.LinkSetUp(veth1) - if err != nil { - return nil, err - } - veth1IP := &netlink.Addr{ IPNet: &net.IPNet{ IP: net.IPv4(192, 168, 0, 4), @@ -125,6 +120,11 @@ func NewVirtualPairing(ns netns.NsHandle, wanLinkName string) (pair *PairLink, e return nil, err } + err = netlink.LinkSetUp(veth1) + if err != nil { + return nil, err + } + // echo 1 > /proc/sys/net/ipv4/conf/veth4bbc9fef/proxy_arp err = sysctl.Set(fmt.Sprintf("net.ipv4.conf.%s.proxy_arp", veth1.Attrs().Name), "1") if err != nil { @@ -161,21 +161,21 @@ func NewVirtualPairing(ns netns.NsHandle, wanLinkName string) (pair *PairLink, e } return &PairLink{ - host: Peer{veth1, origins, *veth1IP.IPNet}, - guest: Peer{veth2, ns, *veth2IP.IPNet}, + Host: Peer{veth1, origins, *veth1IP.IPNet}, + Guest: Peer{veth2, ns, *veth2IP.IPNet}, wanLinkName: wanLinkName, }, nil } -// DeleteVirtualPairing removes an interface link. +// DeleteVirtualPairing removes an interface Link. func (p *PairLink) DeleteVirtualPairing() error { - err := netns.Set(p.host.netNs) + err := netns.Set(p.Host.NetNs) if err != nil { - return fmt.Errorf("set original Ns for deleting veth %q: %v", p.host.link.Attrs().Name, err) + return fmt.Errorf("set original Ns for deleting veth %q: %v", p.Host.Link.Attrs().Name, err) } - if err := netlink.LinkDel(p.host.link); err != nil { - return fmt.Errorf("failed to delete %q: %v", p.host.link.Attrs().Name, err) + if err := netlink.LinkDel(p.Host.Link); err != nil { + return fmt.Errorf("failed to delete %q: %v", p.Host.Link.Attrs().Name, err) } ipt, err := iptables.New() @@ -184,13 +184,13 @@ func (p *PairLink) DeleteVirtualPairing() error { } // sudo iptables -t nat -I POSTROUTING 1 -o wlp3s0 -j MASQUERADE --source 192.168.0.3/24 - err = ipt.DeleteIfExists("nat", "POSTROUTING", "-o", p.wanLinkName, "-j", "MASQUERADE", "--source", p.guest.ip.String()) + err = ipt.DeleteIfExists("nat", "POSTROUTING", "-o", p.wanLinkName, "-j", "MASQUERADE", "--source", p.Guest.Ip.String()) if err != nil { return err } // sudo iptables -I FORWARD 1 -o vethee574a50 -i wlp3s0 -j ACCEPT - err = ipt.DeleteIfExists("filter", "FORWARD", "-o", p.host.link.Attrs().Name, "-i", p.wanLinkName, "-j", "ACCEPT") + err = ipt.DeleteIfExists("filter", "FORWARD", "-o", p.Host.Link.Attrs().Name, "-i", p.wanLinkName, "-j", "ACCEPT") if err != nil { return err }