From 01970282204170183dabd6243a35001023a03839 Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Wed, 3 Jun 2015 09:36:47 -0700 Subject: [PATCH 1/3] Make GenerateIfaceName generic Currently GenerateIfaceName is defined in bridge.go and it specifically tries to only generate an interface name only with `veth` prefix. Make it generic so that it can accept a prefix and length of random bytes. Also move it to netutils since it is useful to generate various kinds of interface names using it. Signed-off-by: Jana Radhakrishnan --- drivers/bridge/bridge.go | 24 ++---------------------- netutils/utils.go | 21 +++++++++++++++++++++ types/types_test.go | 5 +++-- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/drivers/bridge/bridge.go b/drivers/bridge/bridge.go index 60077d7..b144e78 100644 --- a/drivers/bridge/bridge.go +++ b/drivers/bridge/bridge.go @@ -5,7 +5,6 @@ import ( "net" "os/exec" "strconv" - "strings" "sync" "github.com/Sirupsen/logrus" @@ -701,13 +700,13 @@ func (d *driver) CreateEndpoint(nid, eid types.UUID, epInfo driverapi.EndpointIn }() // Generate a name for what will be the host side pipe interface - name1, err := generateIfaceName() + name1, err := netutils.GenerateIfaceName(vethPrefix, vethLen) if err != nil { return err } // Generate a name for what will be the sandbox side pipe interface - name2, err := generateIfaceName() + name2, err := netutils.GenerateIfaceName(vethPrefix, vethLen) if err != nil { return err } @@ -1184,22 +1183,3 @@ func electMacAddress(epConfig *endpointConfiguration) net.HardwareAddr { } return netutils.GenerateRandomMAC() } - -// Generates a name to be used for a virtual ethernet -// interface. The name is constructed by 'veth' appended -// by a randomly generated hex value. (example: veth0f60e2c) -func generateIfaceName() (string, error) { - for i := 0; i < 3; i++ { - name, err := netutils.GenerateRandomName(vethPrefix, vethLen) - if err != nil { - continue - } - if _, err := net.InterfaceByName(name); err != nil { - if strings.Contains(err.Error(), "no such") { - return name, nil - } - return "", err - } - } - return "", &ErrIfaceName{} -} diff --git a/netutils/utils.go b/netutils/utils.go index 98da12e..222b7a2 100644 --- a/netutils/utils.go +++ b/netutils/utils.go @@ -9,7 +9,9 @@ import ( "fmt" "io" "net" + "strings" + "github.com/docker/libnetwork/types" "github.com/vishvananda/netlink" ) @@ -147,3 +149,22 @@ func GenerateRandomName(prefix string, size int) (string, error) { } return prefix + hex.EncodeToString(id)[:size], nil } + +// GenerateIfaceName returns an interface name using the passed in +// prefix and the length of random bytes. The api ensures that the +// there are is no interface which exists with that name. +func GenerateIfaceName(prefix string, len int) (string, error) { + for i := 0; i < 3; i++ { + name, err := GenerateRandomName(prefix, len) + if err != nil { + continue + } + if _, err := net.InterfaceByName(name); err != nil { + if strings.Contains(err.Error(), "no such") { + return name, nil + } + return "", err + } + } + return "", types.InternalErrorf("could not generate interface name") +} diff --git a/types/types_test.go b/types/types_test.go index 9e96ea8..15a58e1 100644 --- a/types/types_test.go +++ b/types/types_test.go @@ -1,11 +1,12 @@ package types import ( + "flag" "testing" - - _ "github.com/docker/libnetwork/netutils" ) +var runningInContainer = flag.Bool("incontainer", false, "Indicates if the test is running in a container") + func TestErrorConstructors(t *testing.T) { var err error From df6ed350aeb696a54c8b9f94914fe965a3e11161 Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Thu, 4 Jun 2015 20:21:23 -0700 Subject: [PATCH 2/3] Refactor sandbox code to use interfaces Currently sandbox code exposes bare structs externally to the package. It is untenable to continue this way and it becomes too inflexible to use it to store internal state. Changed all of them to use interfaces. Also cleaned up a lot of boiler plate code which needs to set into namespace. Signed-off-by: Jana Radhakrishnan --- drivers/bridge/bridge.go | 34 ++-- drivers/bridge/port_mapping.go | 5 +- sandbox/configure_linux.go | 176 -------------------- sandbox/interface_linux.go | 284 +++++++++++++++++++++++++++++++++ sandbox/namespace_linux.go | 241 ++++------------------------ sandbox/options_linux.go | 29 ++++ sandbox/route_linux.go | 198 +++++++++++++++++++++++ sandbox/sandbox.go | 178 +++++---------------- sandbox/sandbox_linux_test.go | 58 ++++--- sandbox/sandbox_test.go | 152 ++++++------------ sandboxdata.go | 21 ++- 11 files changed, 681 insertions(+), 695 deletions(-) delete mode 100644 sandbox/configure_linux.go create mode 100644 sandbox/interface_linux.go create mode 100644 sandbox/options_linux.go create mode 100644 sandbox/route_linux.go diff --git a/drivers/bridge/bridge.go b/drivers/bridge/bridge.go index b144e78..dde595a 100644 --- a/drivers/bridge/bridge.go +++ b/drivers/bridge/bridge.go @@ -14,7 +14,6 @@ import ( "github.com/docker/libnetwork/netutils" "github.com/docker/libnetwork/options" "github.com/docker/libnetwork/portmapper" - "github.com/docker/libnetwork/sandbox" "github.com/docker/libnetwork/types" "github.com/vishvananda/netlink" ) @@ -70,7 +69,9 @@ type containerConfiguration struct { type bridgeEndpoint struct { id types.UUID - intf *sandbox.Interface + srcName string + addr *net.IPNet + addrv6 *net.IPNet macAddress net.HardwareAddr config *endpointConfiguration // User specified parameters containerConfig *containerConfiguration @@ -802,25 +803,20 @@ func (d *driver) CreateEndpoint(nid, eid types.UUID, epInfo driverapi.EndpointIn } // Create the sandbox side pipe interface - intf := &sandbox.Interface{} - intf.SrcName = name2 - intf.DstName = containerVethPrefix - intf.Address = ipv4Addr + endpoint.srcName = name2 + endpoint.addr = ipv4Addr if config.EnableIPv6 { - intf.AddressIPv6 = ipv6Addr + endpoint.addrv6 = ipv6Addr } - // Store the interface in endpoint, this is needed for cleanup on DeleteEndpoint() - endpoint.intf = intf - err = epInfo.AddInterface(ifaceID, endpoint.macAddress, *ipv4Addr, *ipv6Addr) if err != nil { return err } // Program any required port mapping and store them in the endpoint - endpoint.portMapping, err = n.allocatePorts(epConfig, intf, config.DefaultBindingIP, config.EnableUserlandProxy) + endpoint.portMapping, err = n.allocatePorts(epConfig, endpoint, config.DefaultBindingIP, config.EnableUserlandProxy) if err != nil { return err } @@ -882,14 +878,14 @@ func (d *driver) DeleteEndpoint(nid, eid types.UUID) error { n.releasePorts(ep) // Release the v4 address allocated to this endpoint's sandbox interface - err = ipAllocator.ReleaseIP(n.bridge.bridgeIPv4, ep.intf.Address.IP) + err = ipAllocator.ReleaseIP(n.bridge.bridgeIPv4, ep.addr.IP) if err != nil { return err } // Release the v6 address allocated to this endpoint's sandbox interface if config.EnableIPv6 { - err := ipAllocator.ReleaseIP(n.bridge.bridgeIPv6, ep.intf.AddressIPv6.IP) + err := ipAllocator.ReleaseIP(n.bridge.bridgeIPv6, ep.addrv6.IP) if err != nil { return err } @@ -897,7 +893,7 @@ func (d *driver) DeleteEndpoint(nid, eid types.UUID) error { // Try removal of link. Discard error: link pair might have // already been deleted by sandbox delete. - link, err := netlink.LinkByName(ep.intf.SrcName) + link, err := netlink.LinkByName(ep.srcName) if err == nil { netlink.LinkDel(link) } @@ -981,7 +977,7 @@ func (d *driver) Join(nid, eid types.UUID, sboxKey string, jinfo driverapi.JoinI for _, iNames := range jinfo.InterfaceNames() { // Make sure to set names on the correct interface ID. if iNames.ID() == ifaceID { - err = iNames.SetNames(endpoint.intf.SrcName, endpoint.intf.DstName) + err = iNames.SetNames(endpoint.srcName, containerVethPrefix) if err != nil { return err } @@ -1059,8 +1055,8 @@ func (d *driver) link(network *bridgeNetwork, endpoint *bridgeEndpoint, options return err } - l := newLink(parentEndpoint.intf.Address.IP.String(), - endpoint.intf.Address.IP.String(), + l := newLink(parentEndpoint.addr.IP.String(), + endpoint.addr.IP.String(), endpoint.config.ExposedPorts, network.config.BridgeName) if enable { err = l.Enable() @@ -1092,8 +1088,8 @@ func (d *driver) link(network *bridgeNetwork, endpoint *bridgeEndpoint, options continue } - l := newLink(endpoint.intf.Address.IP.String(), - childEndpoint.intf.Address.IP.String(), + l := newLink(endpoint.addr.IP.String(), + childEndpoint.addr.IP.String(), childEndpoint.config.ExposedPorts, network.config.BridgeName) if enable { err = l.Enable() diff --git a/drivers/bridge/port_mapping.go b/drivers/bridge/port_mapping.go index 8f37795..b102132 100644 --- a/drivers/bridge/port_mapping.go +++ b/drivers/bridge/port_mapping.go @@ -7,7 +7,6 @@ import ( "net" "github.com/Sirupsen/logrus" - "github.com/docker/libnetwork/sandbox" "github.com/docker/libnetwork/types" ) @@ -15,7 +14,7 @@ var ( defaultBindingIP = net.IPv4(0, 0, 0, 0) ) -func (n *bridgeNetwork) allocatePorts(epConfig *endpointConfiguration, intf *sandbox.Interface, reqDefBindIP net.IP, ulPxyEnabled bool) ([]types.PortBinding, error) { +func (n *bridgeNetwork) allocatePorts(epConfig *endpointConfiguration, ep *bridgeEndpoint, reqDefBindIP net.IP, ulPxyEnabled bool) ([]types.PortBinding, error) { if epConfig == nil || epConfig.PortBindings == nil { return nil, nil } @@ -25,7 +24,7 @@ func (n *bridgeNetwork) allocatePorts(epConfig *endpointConfiguration, intf *san defHostIP = reqDefBindIP } - return n.allocatePortsInternal(epConfig.PortBindings, intf.Address.IP, defHostIP, ulPxyEnabled) + return n.allocatePortsInternal(epConfig.PortBindings, ep.addr.IP, defHostIP, ulPxyEnabled) } func (n *bridgeNetwork) allocatePortsInternal(bindings []types.PortBinding, containerIP, defHostIP net.IP, ulPxyEnabled bool) ([]types.PortBinding, error) { diff --git a/sandbox/configure_linux.go b/sandbox/configure_linux.go deleted file mode 100644 index 4022170..0000000 --- a/sandbox/configure_linux.go +++ /dev/null @@ -1,176 +0,0 @@ -package sandbox - -import ( - "fmt" - "net" - "os" - "runtime" - - "github.com/vishvananda/netlink" - "github.com/vishvananda/netns" -) - -func configureInterface(iface netlink.Link, settings *Interface) error { - ifaceName := iface.Attrs().Name - ifaceConfigurators := []struct { - Fn func(netlink.Link, *Interface) error - ErrMessage string - }{ - {setInterfaceName, fmt.Sprintf("error renaming interface %q to %q", ifaceName, settings.DstName)}, - {setInterfaceIP, fmt.Sprintf("error setting interface %q IP to %q", ifaceName, settings.Address)}, - {setInterfaceIPv6, fmt.Sprintf("error setting interface %q IPv6 to %q", ifaceName, settings.AddressIPv6)}, - {setInterfaceRoutes, fmt.Sprintf("error setting interface %q routes to %q", ifaceName, settings.Routes)}, - } - - for _, config := range ifaceConfigurators { - if err := config.Fn(iface, settings); err != nil { - return fmt.Errorf("%s: %v", config.ErrMessage, err) - } - } - return nil -} - -func programGateway(path string, gw net.IP, isAdd bool) error { - runtime.LockOSThread() - defer runtime.UnlockOSThread() - - origns, err := netns.Get() - if err != nil { - return err - } - defer origns.Close() - - f, err := os.OpenFile(path, os.O_RDONLY, 0) - if err != nil { - return fmt.Errorf("failed get network namespace %q: %v", path, err) - } - defer f.Close() - - nsFD := f.Fd() - if err = netns.Set(netns.NsHandle(nsFD)); err != nil { - return err - } - defer netns.Set(origns) - - gwRoutes, err := netlink.RouteGet(gw) - if err != nil { - return fmt.Errorf("route for the gateway could not be found: %v", err) - } - - if isAdd { - return netlink.RouteAdd(&netlink.Route{ - Scope: netlink.SCOPE_UNIVERSE, - LinkIndex: gwRoutes[0].LinkIndex, - Gw: gw, - }) - } - - return netlink.RouteDel(&netlink.Route{ - Scope: netlink.SCOPE_UNIVERSE, - LinkIndex: gwRoutes[0].LinkIndex, - Gw: gw, - }) -} - -// Program a route in to the namespace routing table. -func programRoute(path string, dest *net.IPNet, nh net.IP) error { - runtime.LockOSThread() - defer runtime.UnlockOSThread() - - origns, err := netns.Get() - if err != nil { - return err - } - defer origns.Close() - - f, err := os.OpenFile(path, os.O_RDONLY, 0) - if err != nil { - return fmt.Errorf("failed get network namespace %q: %v", path, err) - } - defer f.Close() - - nsFD := f.Fd() - if err = netns.Set(netns.NsHandle(nsFD)); err != nil { - return err - } - defer netns.Set(origns) - - gwRoutes, err := netlink.RouteGet(nh) - if err != nil { - return fmt.Errorf("route for the next hop could not be found: %v", err) - } - - return netlink.RouteAdd(&netlink.Route{ - Scope: netlink.SCOPE_UNIVERSE, - LinkIndex: gwRoutes[0].LinkIndex, - Gw: gwRoutes[0].Gw, - Dst: dest, - }) -} - -// Delete a route from the namespace routing table. -func removeRoute(path string, dest *net.IPNet, nh net.IP) error { - runtime.LockOSThread() - defer runtime.UnlockOSThread() - - origns, err := netns.Get() - if err != nil { - return err - } - defer origns.Close() - - f, err := os.OpenFile(path, os.O_RDONLY, 0) - if err != nil { - return fmt.Errorf("failed get network namespace %q: %v", path, err) - } - defer f.Close() - - nsFD := f.Fd() - if err = netns.Set(netns.NsHandle(nsFD)); err != nil { - return err - } - defer netns.Set(origns) - - gwRoutes, err := netlink.RouteGet(nh) - if err != nil { - return fmt.Errorf("route for the next hop could not be found: %v", err) - } - - return netlink.RouteDel(&netlink.Route{ - Scope: netlink.SCOPE_UNIVERSE, - LinkIndex: gwRoutes[0].LinkIndex, - Gw: gwRoutes[0].Gw, - Dst: dest, - }) -} - -func setInterfaceIP(iface netlink.Link, settings *Interface) error { - ipAddr := &netlink.Addr{IPNet: settings.Address, Label: ""} - return netlink.AddrAdd(iface, ipAddr) -} - -func setInterfaceIPv6(iface netlink.Link, settings *Interface) error { - if settings.AddressIPv6 == nil { - return nil - } - ipAddr := &netlink.Addr{IPNet: settings.AddressIPv6, Label: ""} - return netlink.AddrAdd(iface, ipAddr) -} - -func setInterfaceName(iface netlink.Link, settings *Interface) error { - return netlink.LinkSetName(iface, settings.DstName) -} - -func setInterfaceRoutes(iface netlink.Link, settings *Interface) error { - for _, route := range settings.Routes { - err := netlink.RouteAdd(&netlink.Route{ - Scope: netlink.SCOPE_LINK, - LinkIndex: iface.Attrs().Index, - Dst: route, - }) - if err != nil { - return err - } - } - return nil -} diff --git a/sandbox/interface_linux.go b/sandbox/interface_linux.go new file mode 100644 index 0000000..0933985 --- /dev/null +++ b/sandbox/interface_linux.go @@ -0,0 +1,284 @@ +package sandbox + +import ( + "fmt" + "net" + "sync" + + "github.com/docker/libnetwork/types" + "github.com/vishvananda/netlink" +) + +// IfaceOption is a function option type to set interface options +type IfaceOption func(i *nwIface) + +type nwIface struct { + srcName string + dstName string + address *net.IPNet + addressIPv6 *net.IPNet + routes []*net.IPNet + ns *networkNamespace + sync.Mutex +} + +func (i *nwIface) SrcName() string { + i.Lock() + defer i.Unlock() + + return i.srcName +} + +func (i *nwIface) DstName() string { + i.Lock() + defer i.Unlock() + + return i.dstName +} + +func (i *nwIface) DstMaster() string { + i.Lock() + defer i.Unlock() + + return i.dstMaster +} + +func (i *nwIface) Bridge() bool { + i.Lock() + defer i.Unlock() + + return i.bridge +} + +func (i *nwIface) Master() string { + i.Lock() + defer i.Unlock() + + return i.master +} + +func (i *nwIface) Address() *net.IPNet { + i.Lock() + defer i.Unlock() + + return types.GetIPNetCopy(i.address) +} + +func (i *nwIface) AddressIPv6() *net.IPNet { + i.Lock() + defer i.Unlock() + + return types.GetIPNetCopy(i.addressIPv6) +} + +func (i *nwIface) Routes() []*net.IPNet { + i.Lock() + defer i.Unlock() + + routes := make([]*net.IPNet, len(i.routes)) + for index, route := range i.routes { + r := types.GetIPNetCopy(route) + routes[index] = r + } + + return routes +} + +func (n *networkNamespace) Interfaces() []Interface { + n.Lock() + defer n.Unlock() + + ifaces := make([]Interface, len(n.iFaces)) + + for i, iface := range n.iFaces { + ifaces[i] = iface + } + + return ifaces +} + +func (i *nwIface) Remove() error { + i.Lock() + n := i.ns + i.Unlock() + + n.Lock() + path := n.path + n.Unlock() + + return nsInvoke(path, func(nsFD int) error { return nil }, func(callerFD int) error { + // Find the network inteerface identified by the DstName attribute. + iface, err := netlink.LinkByName(i.DstName()) + if err != nil { + return err + } + + // Down the interface before configuring + if err := netlink.LinkSetDown(iface); err != nil { + return err + } + + err = netlink.LinkSetName(iface, i.SrcName()) + if err != nil { + fmt.Println("LinkSetName failed: ", err) + return err + } + + // if it is a bridge just delete it. + if i.Bridge() { + if err := netlink.LinkDel(iface); err != nil { + return fmt.Errorf("failed deleting bridge %q: %v", i.SrcName(), err) + } + } else { + // Move the network interface to caller namespace. + if err := netlink.LinkSetNsFd(iface, callerFD); err != nil { + fmt.Println("LinkSetNsPid failed: ", err) + return err + } + } + + n.Lock() + for index, intf := range n.iFaces { + if intf == i { + n.iFaces = append(n.iFaces[:index], n.iFaces[index+1:]...) + break + } + } + n.Unlock() + + return nil + }) +} + +func (n *networkNamespace) findDstMaster(srcName string) string { + n.Lock() + defer n.Unlock() + + for _, i := range n.iFaces { + // The master should match the srcname of the interface and the + // master interface should be of type bridge. + if i.SrcName() == srcName && i.Bridge() { + return i.DstName() + } + } + + return "" +} + +func (n *networkNamespace) AddInterface(srcName, dstPrefix string, options ...IfaceOption) error { + i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n} + i.processInterfaceOptions(options...) + + n.Lock() + i.dstName = fmt.Sprintf("%s%d", i.dstName, n.nextIfIndex) + n.nextIfIndex++ + path := n.path + n.Unlock() + + return nsInvoke(path, func(nsFD int) error { + // Find the network interface identified by the SrcName attribute. + iface, err := netlink.LinkByName(i.srcName) + if err != nil { + return fmt.Errorf("failed to get link by name %q: %v", i.srcName, err) + } + + // Move the network interface to the destination namespace. + if err := netlink.LinkSetNsFd(iface, nsFD); err != nil { + return fmt.Errorf("failed to set namespace on link %q: %v", i.srcName, err) + } + + return nil + }, func(callerFD int) error { + // Find the network interface identified by the SrcName attribute. + iface, err := netlink.LinkByName(i.srcName) + if err != nil { + return fmt.Errorf("failed to get link by name %q: %v", i.srcName, err) + } + + // Down the interface before configuring + if err := netlink.LinkSetDown(iface); err != nil { + return fmt.Errorf("failed to set link down: %v", err) + } + + // Configure the interface now this is moved in the proper namespace. + if err := configureInterface(iface, i); err != nil { + return err + } + + // Up the interface. + if err := netlink.LinkSetUp(iface); err != nil { + return fmt.Errorf("failed to set link up: %v", err) + } + + n.Lock() + n.iFaces = append(n.iFaces, i) + n.Unlock() + + return nil + }) +} + +func configureInterface(iface netlink.Link, i *nwIface) error { + ifaceName := iface.Attrs().Name + ifaceConfigurators := []struct { + Fn func(netlink.Link, *nwIface) error + ErrMessage string + }{ + {setInterfaceName, fmt.Sprintf("error renaming interface %q to %q", ifaceName, i.DstName())}, + {setInterfaceIP, fmt.Sprintf("error setting interface %q IP to %q", ifaceName, i.Address())}, + {setInterfaceIPv6, fmt.Sprintf("error setting interface %q IPv6 to %q", ifaceName, i.AddressIPv6())}, + {setInterfaceRoutes, fmt.Sprintf("error setting interface %q routes to %q", ifaceName, i.Routes())}, + {setInterfaceMaster, fmt.Sprintf("error setting interface %q master to %q", ifaceName, i.DstMaster())}, + } + + for _, config := range ifaceConfigurators { + if err := config.Fn(iface, i); err != nil { + return fmt.Errorf("%s: %v", config.ErrMessage, err) + } + } + return nil +} + +func setInterfaceMaster(iface netlink.Link, i *nwIface) error { + if i.DstMaster() == "" { + return nil + } + + return netlink.LinkSetMaster(iface, &netlink.Bridge{ + LinkAttrs: netlink.LinkAttrs{Name: i.DstMaster()}}) +} + +func setInterfaceIP(iface netlink.Link, i *nwIface) error { + if i.Address() == nil { + return nil + } + + ipAddr := &netlink.Addr{IPNet: i.Address(), Label: ""} + return netlink.AddrAdd(iface, ipAddr) +} + +func setInterfaceIPv6(iface netlink.Link, i *nwIface) error { + if i.AddressIPv6() == nil { + return nil + } + ipAddr := &netlink.Addr{IPNet: i.AddressIPv6(), Label: ""} + return netlink.AddrAdd(iface, ipAddr) +} + +func setInterfaceName(iface netlink.Link, i *nwIface) error { + return netlink.LinkSetName(iface, i.DstName()) +} + +func setInterfaceRoutes(iface netlink.Link, i *nwIface) error { + for _, route := range i.Routes() { + err := netlink.RouteAdd(&netlink.Route{ + Scope: netlink.SCOPE_LINK, + LinkIndex: iface.Attrs().Index, + Dst: route, + }) + if err != nil { + return err + } + } + return nil +} diff --git a/sandbox/namespace_linux.go b/sandbox/namespace_linux.go index 3d8a98c..1b9b7fa 100644 --- a/sandbox/namespace_linux.go +++ b/sandbox/namespace_linux.go @@ -32,9 +32,12 @@ var ( // interface. It represents a linux network namespace, and moves an interface // into it when called on method AddInterface or sets the gateway etc. type networkNamespace struct { - path string - sinfo *Info - nextIfIndex int + path string + iFaces []*nwIface + gw net.IP + gwv6 net.IP + staticRoutes []*types.StaticRoute + nextIfIndex int sync.Mutex } @@ -127,12 +130,16 @@ func GenerateKey(containerID string) string { // NewSandbox provides a new sandbox instance created in an os specific way // provided a key which uniquely identifies the sandbox func NewSandbox(key string, osCreate bool) (Sandbox, error) { - info, err := createNetworkNamespace(key, osCreate) + err := createNetworkNamespace(key, osCreate) if err != nil { return nil, err } - return &networkNamespace{path: key, sinfo: info}, nil + return &networkNamespace{path: key}, nil +} + +func (n *networkNamespace) InterfaceOptions() IfaceOptionSetter { + return n } func reexecCreateNamespace() { @@ -149,18 +156,18 @@ func reexecCreateNamespace() { } } -func createNetworkNamespace(path string, osCreate bool) (*Info, error) { +func createNetworkNamespace(path string, osCreate bool) error { runtime.LockOSThread() defer runtime.UnlockOSThread() origns, err := netns.Get() if err != nil { - return nil, err + return err } defer origns.Close() if err := createNamespaceFile(path); err != nil { - return nil, err + return err } cmd := &exec.Cmd{ @@ -174,12 +181,10 @@ func createNetworkNamespace(path string, osCreate bool) (*Info, error) { cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET } if err := cmd.Run(); err != nil { - return nil, fmt.Errorf("namespace creation reexec command failed: %v", err) + return fmt.Errorf("namespace creation reexec command failed: %v", err) } - interfaces := []*Interface{} - info := &Info{Interfaces: interfaces} - return info, nil + return nil } func unmountNamespaceFile(path string) { @@ -217,7 +222,7 @@ func loopbackUp() error { return netlink.LinkSetUp(iface) } -func (n *networkNamespace) RemoveInterface(i *Interface) error { +func nsInvoke(path string, prefunc func(nsFD int) error, postfunc func(callerFD int) error) error { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -227,84 +232,18 @@ func (n *networkNamespace) RemoveInterface(i *Interface) error { } defer origns.Close() - f, err := os.OpenFile(n.path, os.O_RDONLY, 0) + f, err := os.OpenFile(path, os.O_RDONLY, 0) if err != nil { - return fmt.Errorf("failed get network namespace %q: %v", n.path, err) + return fmt.Errorf("failed get network namespace %q: %v", path, err) } defer f.Close() nsFD := f.Fd() - if err = netns.Set(netns.NsHandle(nsFD)); err != nil { - return err - } - defer netns.Set(origns) - // Find the network inteerface identified by the DstName attribute. - iface, err := netlink.LinkByName(i.DstName) - if err != nil { - return err - } - - // Down the interface before configuring - if err := netlink.LinkSetDown(iface); err != nil { - return err - } - - err = netlink.LinkSetName(iface, i.SrcName) - if err != nil { - fmt.Println("LinkSetName failed: ", err) - return err - } - - // Move the network interface to caller namespace. - if err := netlink.LinkSetNsFd(iface, int(origns)); err != nil { - fmt.Println("LinkSetNsPid failed: ", err) - return err - } - - n.Lock() - for index, intf := range n.sinfo.Interfaces { - if intf == i { - n.sinfo.Interfaces = append(n.sinfo.Interfaces[:index], n.sinfo.Interfaces[index+1:]...) - break - } - } - n.Unlock() - - return nil -} - -func (n *networkNamespace) AddInterface(i *Interface) error { - n.Lock() - i.DstName = fmt.Sprintf("%s%d", i.DstName, n.nextIfIndex) - n.nextIfIndex++ - n.Unlock() - - runtime.LockOSThread() - defer runtime.UnlockOSThread() - - origns, err := netns.Get() - if err != nil { - return err - } - defer origns.Close() - - f, err := os.OpenFile(n.path, os.O_RDONLY, 0) - if err != nil { - return fmt.Errorf("failed get network namespace %q: %v", n.path, err) - } - defer f.Close() - - // Find the network interface identified by the SrcName attribute. - iface, err := netlink.LinkByName(i.SrcName) - if err != nil { - return err - } - - // Move the network interface to the destination namespace. - nsFD := f.Fd() - if err := netlink.LinkSetNsFd(iface, int(nsFD)); err != nil { - return err + // Invoked before the namespace switch happens but after the namespace file + // handle is obtained. + if err := prefunc(int(nsFD)); err != nil { + return fmt.Errorf("failed in prefunc: %v", err) } if err = netns.Set(netns.NsHandle(nsFD)); err != nil { @@ -312,133 +251,19 @@ func (n *networkNamespace) AddInterface(i *Interface) error { } defer netns.Set(origns) - // Down the interface before configuring - if err := netlink.LinkSetDown(iface); err != nil { - return err - } - - // Configure the interface now this is moved in the proper namespace. - if err := configureInterface(iface, i); err != nil { - return err - } - - // Up the interface. - if err := netlink.LinkSetUp(iface); err != nil { - return err - } - - n.Lock() - n.sinfo.Interfaces = append(n.sinfo.Interfaces, i) - n.Unlock() - - return nil + // Invoked after the namespace switch. + return postfunc(int(origns)) } -func (n *networkNamespace) SetGateway(gw net.IP) error { - // Silently return if the gateway is empty - if len(gw) == 0 { - return nil - } - - err := programGateway(n.path, gw, true) - if err == nil { - n.Lock() - n.sinfo.Gateway = gw - n.Unlock() - } - - return err -} - -func (n *networkNamespace) UnsetGateway() error { - n.Lock() - gw := n.sinfo.Gateway - n.Unlock() - - // Silently return if the gateway is empty - if len(gw) == 0 { - return nil - } - - err := programGateway(n.path, gw, false) - if err == nil { - n.Lock() - n.sinfo.Gateway = net.IP{} - n.Unlock() - } - - return err -} - -func (n *networkNamespace) SetGatewayIPv6(gw net.IP) error { - // Silently return if the gateway is empty - if len(gw) == 0 { - return nil - } - - err := programGateway(n.path, gw, true) - if err == nil { - n.Lock() - n.sinfo.GatewayIPv6 = gw - n.Unlock() - } - - return err -} - -func (n *networkNamespace) UnsetGatewayIPv6() error { - n.Lock() - gw := n.sinfo.GatewayIPv6 - n.Unlock() - - // Silently return if the gateway is empty - if len(gw) == 0 { - return nil - } - - err := programGateway(n.path, gw, false) - if err == nil { - n.Lock() - n.sinfo.GatewayIPv6 = net.IP{} - n.Unlock() - } - - return err -} - -func (n *networkNamespace) AddStaticRoute(r *types.StaticRoute) error { - err := programRoute(n.path, r.Destination, r.NextHop) - if err == nil { - n.Lock() - n.sinfo.StaticRoutes = append(n.sinfo.StaticRoutes, r) - n.Unlock() - } - return err -} - -func (n *networkNamespace) RemoveStaticRoute(r *types.StaticRoute) error { - err := removeRoute(n.path, r.Destination, r.NextHop) - if err == nil { - n.Lock() - lastIndex := len(n.sinfo.StaticRoutes) - 1 - for i, v := range n.sinfo.StaticRoutes { - if v == r { - // Overwrite the route we're removing with the last element - n.sinfo.StaticRoutes[i] = n.sinfo.StaticRoutes[lastIndex] - // Shorten the slice to trim the extra element - n.sinfo.StaticRoutes = n.sinfo.StaticRoutes[:lastIndex] - break - } - } - n.Unlock() - } - return err -} - -func (n *networkNamespace) Interfaces() []*Interface { +func (n *networkNamespace) nsPath() string { n.Lock() defer n.Unlock() - return n.sinfo.Interfaces + + return n.path +} + +func (n *networkNamespace) Info() Info { + return n } func (n *networkNamespace) Key() string { diff --git a/sandbox/options_linux.go b/sandbox/options_linux.go new file mode 100644 index 0000000..40199e0 --- /dev/null +++ b/sandbox/options_linux.go @@ -0,0 +1,29 @@ +package sandbox + +import "net" + +func (i *nwIface) processInterfaceOptions(options ...IfaceOption) { + for _, opt := range options { + if opt != nil { + opt(i) + } + } +} + +func (n *networkNamespace) Address(addr *net.IPNet) IfaceOption { + return func(i *nwIface) { + i.address = addr + } +} + +func (n *networkNamespace) AddressIPv6(addr *net.IPNet) IfaceOption { + return func(i *nwIface) { + i.addressIPv6 = addr + } +} + +func (n *networkNamespace) Routes(routes []*net.IPNet) IfaceOption { + return func(i *nwIface) { + i.routes = routes + } +} diff --git a/sandbox/route_linux.go b/sandbox/route_linux.go new file mode 100644 index 0000000..8326514 --- /dev/null +++ b/sandbox/route_linux.go @@ -0,0 +1,198 @@ +package sandbox + +import ( + "fmt" + "net" + + "github.com/docker/libnetwork/types" + "github.com/vishvananda/netlink" +) + +func (n *networkNamespace) Gateway() net.IP { + n.Lock() + defer n.Unlock() + + return n.gw +} + +func (n *networkNamespace) GatewayIPv6() net.IP { + n.Lock() + defer n.Unlock() + + return n.gwv6 +} + +func (n *networkNamespace) StaticRoutes() []*types.StaticRoute { + n.Lock() + defer n.Unlock() + + routes := make([]*types.StaticRoute, len(n.staticRoutes)) + for i, route := range n.staticRoutes { + r := route.GetCopy() + routes[i] = r + } + + return routes +} + +func (n *networkNamespace) setGateway(gw net.IP) { + n.Lock() + n.gw = gw + n.Unlock() +} + +func (n *networkNamespace) setGatewayIPv6(gwv6 net.IP) { + n.Lock() + n.gwv6 = gwv6 + n.Unlock() +} + +func (n *networkNamespace) SetGateway(gw net.IP) error { + // Silently return if the gateway is empty + if len(gw) == 0 { + return nil + } + + err := programGateway(n.nsPath(), gw, true) + if err == nil { + n.setGateway(gw) + } + + return err +} + +func (n *networkNamespace) UnsetGateway() error { + gw := n.Gateway() + + // Silently return if the gateway is empty + if len(gw) == 0 { + return nil + } + + err := programGateway(n.nsPath(), gw, false) + if err == nil { + n.setGateway(net.IP{}) + } + + return err +} + +func programGateway(path string, gw net.IP, isAdd bool) error { + return nsInvoke(path, func(nsFD int) error { return nil }, func(callerFD int) error { + gwRoutes, err := netlink.RouteGet(gw) + if err != nil { + return fmt.Errorf("route for the gateway could not be found: %v", err) + } + + if isAdd { + return netlink.RouteAdd(&netlink.Route{ + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: gwRoutes[0].LinkIndex, + Gw: gw, + }) + } + + return netlink.RouteDel(&netlink.Route{ + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: gwRoutes[0].LinkIndex, + Gw: gw, + }) + }) +} + +// Program a route in to the namespace routing table. +func programRoute(path string, dest *net.IPNet, nh net.IP) error { + return nsInvoke(path, func(nsFD int) error { return nil }, func(callerFD int) error { + gwRoutes, err := netlink.RouteGet(nh) + if err != nil { + return fmt.Errorf("route for the next hop could not be found: %v", err) + } + + return netlink.RouteAdd(&netlink.Route{ + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: gwRoutes[0].LinkIndex, + Gw: gwRoutes[0].Gw, + Dst: dest, + }) + }) +} + +// Delete a route from the namespace routing table. +func removeRoute(path string, dest *net.IPNet, nh net.IP) error { + return nsInvoke(path, func(nsFD int) error { return nil }, func(callerFD int) error { + gwRoutes, err := netlink.RouteGet(nh) + if err != nil { + return fmt.Errorf("route for the next hop could not be found: %v", err) + } + + return netlink.RouteDel(&netlink.Route{ + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: gwRoutes[0].LinkIndex, + Gw: gwRoutes[0].Gw, + Dst: dest, + }) + }) +} + +func (n *networkNamespace) SetGatewayIPv6(gwv6 net.IP) error { + // Silently return if the gateway is empty + if len(gwv6) == 0 { + return nil + } + + err := programGateway(n.nsPath(), gwv6, true) + if err == nil { + n.SetGatewayIPv6(gwv6) + } + + return err +} + +func (n *networkNamespace) UnsetGatewayIPv6() error { + gwv6 := n.GatewayIPv6() + + // Silently return if the gateway is empty + if len(gwv6) == 0 { + return nil + } + + err := programGateway(n.nsPath(), gwv6, false) + if err == nil { + n.Lock() + n.gwv6 = net.IP{} + n.Unlock() + } + + return err +} + +func (n *networkNamespace) AddStaticRoute(r *types.StaticRoute) error { + err := programRoute(n.nsPath(), r.Destination, r.NextHop) + if err == nil { + n.Lock() + n.staticRoutes = append(n.staticRoutes, r) + n.Unlock() + } + return err +} + +func (n *networkNamespace) RemoveStaticRoute(r *types.StaticRoute) error { + n.Lock() + + err := removeRoute(n.nsPath(), r.Destination, r.NextHop) + if err == nil { + n.Lock() + lastIndex := len(n.staticRoutes) - 1 + for i, v := range n.staticRoutes { + if v == r { + // Overwrite the route we're removing with the last element + n.staticRoutes[i] = n.staticRoutes[lastIndex] + // Shorten the slice to trim the extra element + n.staticRoutes = n.staticRoutes[:lastIndex] + break + } + } + n.Unlock() + } + return err +} diff --git a/sandbox/sandbox.go b/sandbox/sandbox.go index e52fba0..92bf4cc 100644 --- a/sandbox/sandbox.go +++ b/sandbox/sandbox.go @@ -12,22 +12,12 @@ type Sandbox interface { // The path where the network namespace is mounted. Key() string - // The collection of Interface previously added with the AddInterface - // method. Note that this doesn't incude network interfaces added in any - // other way (such as the default loopback interface which are automatically - // created on creation of a sandbox). - Interfaces() []*Interface - // Add an existing Interface to this sandbox. The operation will rename // from the Interface SrcName to DstName as it moves, and reconfigure the // interface according to the specified settings. The caller is expected // to only provide a prefix for DstName. The AddInterface api will auto-generate // an appropriate suffix for the DstName to disambiguate. - AddInterface(*Interface) error - - // Remove an interface from the sandbox by renaming to original name - // and moving it out of the sandbox. - RemoveInterface(*Interface) error + AddInterface(SrcName string, DstPrefix string, options ...IfaceOption) error // Set default IPv4 gateway for the sandbox SetGateway(gw net.IP) error @@ -47,25 +37,47 @@ type Sandbox interface { // Remove a static route from the sandbox. RemoveStaticRoute(*types.StaticRoute) error + // Returns an interface with methods to set interface options. + InterfaceOptions() IfaceOptionSetter + + // Returns an interface with methods to get sandbox state. + Info() Info + // Destroy the sandbox Destroy() error } +// IfaceOptionSetter interface defines the option setter methods for interface options. +type IfaceOptionSetter interface { + // Address returns an option setter to set IPv4 address. + Address(*net.IPNet) IfaceOption + + // Address returns an option setter to set IPv6 address. + AddressIPv6(*net.IPNet) IfaceOption + + // Address returns an option setter to set interface routes. + Routes([]*net.IPNet) IfaceOption +} + // Info represents all possible information that // the driver wants to place in the sandbox which includes // interfaces, routes and gateway -type Info struct { - Interfaces []*Interface +type Info interface { + // The collection of Interface previously added with the AddInterface + // method. Note that this doesn't incude network interfaces added in any + // other way (such as the default loopback interface which are automatically + // created on creation of a sandbox). + Interfaces() []Interface // IPv4 gateway for the sandbox. - Gateway net.IP + Gateway() net.IP // IPv6 gateway for the sandbox. - GatewayIPv6 net.IP + GatewayIPv6() net.IP // Additional static routes for the sandbox. (Note that directly // connected routes are stored on the particular interface they refer to.) - StaticRoutes []*types.StaticRoute + StaticRoutes() []*types.StaticRoute // TODO: Add ip tables etc. } @@ -75,140 +87,26 @@ type Info struct { // caller to use this information when moving interface SrcName from host // namespace to DstName in a different net namespace with the appropriate // network settings. -type Interface struct { +type Interface interface { // The name of the interface in the origin network namespace. - SrcName string + SrcName() string // The name that will be assigned to the interface once moves inside a // network namespace. When the caller passes in a DstName, it is only // expected to pass a prefix. The name will modified with an appropriately // auto-generated suffix. - DstName string + DstName() string // IPv4 address for the interface. - Address *net.IPNet + Address() *net.IPNet // IPv6 address for the interface. - AddressIPv6 *net.IPNet + AddressIPv6() *net.IPNet // IP routes for the interface. - Routes []*net.IPNet -} - -// GetCopy returns a copy of this Interface structure -func (i *Interface) GetCopy() *Interface { - copiedRoutes := make([]*net.IPNet, len(i.Routes)) - - for index := range i.Routes { - copiedRoutes[index] = types.GetIPNetCopy(i.Routes[index]) - } - - return &Interface{ - SrcName: i.SrcName, - DstName: i.DstName, - Address: types.GetIPNetCopy(i.Address), - AddressIPv6: types.GetIPNetCopy(i.AddressIPv6), - Routes: copiedRoutes, - } -} - -// Equal checks if this instance of Interface is equal to the passed one -func (i *Interface) Equal(o *Interface) bool { - if i == o { - return true - } - - if o == nil { - return false - } - - if i.SrcName != o.SrcName || i.DstName != o.DstName { - return false - } - - if !types.CompareIPNet(i.Address, o.Address) { - return false - } - - if !types.CompareIPNet(i.AddressIPv6, o.AddressIPv6) { - return false - } - - if len(i.Routes) != len(o.Routes) { - return false - } - - for index := range i.Routes { - if !types.CompareIPNet(i.Routes[index], o.Routes[index]) { - return false - } - } - - return true -} - -// GetCopy returns a copy of this SandboxInfo structure -func (s *Info) GetCopy() *Info { - list := make([]*Interface, len(s.Interfaces)) - for i, iface := range s.Interfaces { - list[i] = iface.GetCopy() - } - gw := types.GetIPCopy(s.Gateway) - gw6 := types.GetIPCopy(s.GatewayIPv6) - - routes := make([]*types.StaticRoute, len(s.StaticRoutes)) - for i, r := range s.StaticRoutes { - routes[i] = r.GetCopy() - } - - return &Info{Interfaces: list, - Gateway: gw, - GatewayIPv6: gw6, - StaticRoutes: routes} -} - -// Equal checks if this instance of SandboxInfo is equal to the passed one -func (s *Info) Equal(o *Info) bool { - if s == o { - return true - } - - if o == nil { - return false - } - - if !s.Gateway.Equal(o.Gateway) { - return false - } - - if !s.GatewayIPv6.Equal(o.GatewayIPv6) { - return false - } - - if (s.Interfaces == nil && o.Interfaces != nil) || - (s.Interfaces != nil && o.Interfaces == nil) || - (len(s.Interfaces) != len(o.Interfaces)) { - return false - } - - // Note: At the moment, the two lists must be in the same order - for i := 0; i < len(s.Interfaces); i++ { - if !s.Interfaces[i].Equal(o.Interfaces[i]) { - return false - } - } - - for index := range s.StaticRoutes { - ss := s.StaticRoutes[index] - oo := o.StaticRoutes[index] - if !types.CompareIPNet(ss.Destination, oo.Destination) { - return false - } - if !ss.NextHop.Equal(oo.NextHop) { - return false - } - } - - return true - + Routes() []*net.IPNet + + // Remove an interface from the sandbox by renaming to original name + // and moving it out of the sandbox. + Remove() error } diff --git a/sandbox/sandbox_linux_test.go b/sandbox/sandbox_linux_test.go index 67175b1..e47ea78 100644 --- a/sandbox/sandbox_linux_test.go +++ b/sandbox/sandbox_linux_test.go @@ -40,7 +40,7 @@ func newKey(t *testing.T) (string, error) { return name, nil } -func newInfo(t *testing.T) (*Info, error) { +func newInfo(t *testing.T) (Sandbox, error) { veth := &netlink.Veth{ LinkAttrs: netlink.LinkAttrs{Name: vethName1, TxQLen: 0}, PeerName: vethName2} @@ -50,31 +50,31 @@ func newInfo(t *testing.T) (*Info, error) { // Store the sandbox side pipe interface // This is needed for cleanup on DeleteEndpoint() - intf1 := &Interface{} - intf1.SrcName = vethName2 - intf1.DstName = sboxIfaceName + intf1 := &nwIface{} + intf1.srcName = vethName2 + intf1.dstName = sboxIfaceName ip4, addr, err := net.ParseCIDR("192.168.1.100/24") if err != nil { return nil, err } - intf1.Address = addr - intf1.Address.IP = ip4 + intf1.address = addr + intf1.address.IP = ip4 // ip6, addrv6, err := net.ParseCIDR("2001:DB8::ABCD/48") ip6, addrv6, err := net.ParseCIDR("fe80::2/64") if err != nil { return nil, err } - intf1.AddressIPv6 = addrv6 - intf1.AddressIPv6.IP = ip6 + intf1.addressIPv6 = addrv6 + intf1.addressIPv6.IP = ip6 _, route, err := net.ParseCIDR("192.168.2.1/32") if err != nil { return nil, err } - intf1.Routes = []*net.IPNet{route} + intf1.routes = []*net.IPNet{route} veth = &netlink.Veth{ LinkAttrs: netlink.LinkAttrs{Name: vethName3, TxQLen: 0}, @@ -84,16 +84,16 @@ func newInfo(t *testing.T) (*Info, error) { return nil, err } - intf2 := &Interface{} - intf2.SrcName = vethName4 - intf2.DstName = sboxIfaceName + intf2 := &nwIface{} + intf2.srcName = vethName4 + intf2.dstName = sboxIfaceName ip4, addr, err = net.ParseCIDR("192.168.2.100/24") if err != nil { return nil, err } - intf2.Address = addr - intf2.Address.IP = ip4 + intf2.address = addr + intf2.address.IP = ip4 // ip6, addrv6, err := net.ParseCIDR("2001:DB8::ABCD/48") ip6, addrv6, err = net.ParseCIDR("fe80::3/64") @@ -101,19 +101,19 @@ func newInfo(t *testing.T) (*Info, error) { if err != nil { return nil, err } - intf2.AddressIPv6 = addrv6 - intf2.AddressIPv6.IP = ip6 + intf2.addressIPv6 = addrv6 + intf2.addressIPv6.IP = ip6 - sinfo := &Info{Interfaces: []*Interface{intf1, intf2}} + info := &networkNamespace{iFaces: []*nwIface{intf1, intf2}} - sinfo.Gateway = net.ParseIP("192.168.1.1") + info.gw = net.ParseIP("192.168.1.1") // sinfo.GatewayIPv6 = net.ParseIP("2001:DB8::1") - sinfo.GatewayIPv6 = net.ParseIP("fe80::1") + info.gwv6 = net.ParseIP("fe80::1") - return sinfo, nil + return info, nil } -func verifySandbox(t *testing.T, s Sandbox) { +func verifySandbox(t *testing.T, s Sandbox, ifaceSuffixes []string) { _, ok := s.(*networkNamespace) if !ok { t.Fatalf("The sandox interface returned is not of type networkNamespace") @@ -140,16 +140,12 @@ func verifySandbox(t *testing.T, s Sandbox) { } defer netns.Set(origns) - _, err = netlink.LinkByName(sboxIfaceName + "0") - if err != nil { - t.Fatalf("Could not find the interface %s inside the sandbox: %v", sboxIfaceName+"0", - err) - } - - _, err = netlink.LinkByName(sboxIfaceName + "1") - if err != nil { - t.Fatalf("Could not find the interface %s inside the sandbox: %v", sboxIfaceName+"1", - err) + for _, suffix := range ifaceSuffixes { + _, err = netlink.LinkByName(sboxIfaceName + suffix) + if err != nil { + t.Fatalf("Could not find the interface %s inside the sandbox: %v", + sboxIfaceName+suffix, err) + } } } diff --git a/sandbox/sandbox_test.go b/sandbox/sandbox_test.go index ca0d6ba..19e87ca 100644 --- a/sandbox/sandbox_test.go +++ b/sandbox/sandbox_test.go @@ -1,11 +1,12 @@ package sandbox import ( - "net" "os" + "runtime" "testing" "github.com/docker/docker/pkg/reexec" + "github.com/docker/libnetwork/netutils" ) func TestMain(m *testing.M) { @@ -16,6 +17,8 @@ func TestMain(m *testing.M) { } func TestSandboxCreate(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + key, err := newKey(t) if err != nil { t.Fatalf("Failed to obtain a key: %v", err) @@ -25,39 +28,49 @@ func TestSandboxCreate(t *testing.T) { if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } + runtime.LockOSThread() if s.Key() != key { t.Fatalf("s.Key() returned %s. Expected %s", s.Key(), key) } - info, err := newInfo(t) + tbox, err := newInfo(t) if err != nil { t.Fatalf("Failed to generate new sandbox info: %v", err) } - for _, i := range info.Interfaces { - err = s.AddInterface(i) + for _, i := range tbox.Info().Interfaces() { + err = s.AddInterface(i.SrcName(), i.DstName(), + tbox.InterfaceOptions().Address(i.Address()), + tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6())) if err != nil { t.Fatalf("Failed to add interfaces to sandbox: %v", err) } + runtime.LockOSThread() } - err = s.SetGateway(info.Gateway) + err = s.SetGateway(tbox.Info().Gateway()) if err != nil { t.Fatalf("Failed to set gateway to sandbox: %v", err) } + runtime.LockOSThread() - err = s.SetGatewayIPv6(info.GatewayIPv6) + err = s.SetGatewayIPv6(tbox.Info().GatewayIPv6()) if err != nil { t.Fatalf("Failed to set ipv6 gateway to sandbox: %v", err) } + runtime.LockOSThread() + + verifySandbox(t, s, []string{"0", "1"}) + runtime.LockOSThread() - verifySandbox(t, s) s.Destroy() verifyCleanup(t, s, true) } func TestSandboxCreateTwice(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + key, err := newKey(t) if err != nil { t.Fatalf("Failed to obtain a key: %v", err) @@ -67,6 +80,7 @@ func TestSandboxCreateTwice(t *testing.T) { if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } + runtime.LockOSThread() // Create another sandbox with the same key to see if we handle it // gracefully. @@ -74,6 +88,7 @@ func TestSandboxCreateTwice(t *testing.T) { if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } + runtime.LockOSThread() s.Destroy() } @@ -95,6 +110,8 @@ func TestSandboxGC(t *testing.T) { } func TestAddRemoveInterface(t *testing.T) { + defer netutils.SetupTestNetNS(t)() + key, err := newKey(t) if err != nil { t.Fatalf("Failed to obtain a key: %v", err) @@ -104,128 +121,49 @@ func TestAddRemoveInterface(t *testing.T) { if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } + runtime.LockOSThread() if s.Key() != key { t.Fatalf("s.Key() returned %s. Expected %s", s.Key(), key) } - info, err := newInfo(t) + tbox, err := newInfo(t) if err != nil { t.Fatalf("Failed to generate new sandbox info: %v", err) } - for _, i := range info.Interfaces { - err = s.AddInterface(i) + for _, i := range tbox.Info().Interfaces() { + err = s.AddInterface(i.SrcName(), i.DstName(), + tbox.InterfaceOptions().Address(i.Address()), + tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6())) if err != nil { t.Fatalf("Failed to add interfaces to sandbox: %v", err) } + runtime.LockOSThread() } - interfaces := s.Interfaces() - if !(interfaces[0].Equal(info.Interfaces[0]) && interfaces[1].Equal(info.Interfaces[1])) { - t.Fatalf("Failed to update Sandbox.sinfo.Interfaces in AddInterfaces") - } + verifySandbox(t, s, []string{"0", "1"}) + runtime.LockOSThread() - if err := s.RemoveInterface(info.Interfaces[0]); err != nil { + interfaces := s.Info().Interfaces() + if err := interfaces[0].Remove(); err != nil { t.Fatalf("Failed to remove interfaces from sandbox: %v", err) } + runtime.LockOSThread() - if !s.Interfaces()[0].Equal(info.Interfaces[1]) { - t.Fatalf("Failed to update the sanbox.sinfo.Interfaces in RemoveInterferce") - } + verifySandbox(t, s, []string{"1"}) + runtime.LockOSThread() - if err := s.AddInterface(info.Interfaces[0]); err != nil { + i := tbox.Info().Interfaces()[0] + if err := s.AddInterface(i.SrcName(), i.DstName(), + tbox.InterfaceOptions().Address(i.Address()), + tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6())); err != nil { t.Fatalf("Failed to add interfaces to sandbox: %v", err) } + runtime.LockOSThread() - interfaces = s.Interfaces() - if !(interfaces[0].Equal(info.Interfaces[1]) && interfaces[1].Equal(info.Interfaces[0])) { - t.Fatalf("Failed to update Sandbox.sinfo.Interfaces in AddInterfaces") - } + verifySandbox(t, s, []string{"1", "2"}) + runtime.LockOSThread() s.Destroy() } - -func TestInterfaceEqual(t *testing.T) { - list := getInterfaceList() - - if !list[0].Equal(list[0]) { - t.Fatalf("Interface.Equal() returned false negative") - } - - if list[0].Equal(list[1]) { - t.Fatalf("Interface.Equal() returned false positive") - } - - if list[0].Equal(list[1]) != list[1].Equal(list[0]) { - t.Fatalf("Interface.Equal() failed commutative check") - } -} - -func TestSandboxInfoEqual(t *testing.T) { - si1 := &Info{Interfaces: getInterfaceList(), Gateway: net.ParseIP("192.168.1.254"), GatewayIPv6: net.ParseIP("2001:2345::abcd:8889")} - si2 := &Info{Interfaces: getInterfaceList(), Gateway: net.ParseIP("172.18.255.254"), GatewayIPv6: net.ParseIP("2001:2345::abcd:8888")} - - if !si1.Equal(si1) { - t.Fatalf("Info.Equal() returned false negative") - } - - if si1.Equal(si2) { - t.Fatalf("Info.Equal() returned false positive") - } - - if si1.Equal(si2) != si2.Equal(si1) { - t.Fatalf("Info.Equal() failed commutative check") - } -} - -func TestInterfaceCopy(t *testing.T) { - for _, iface := range getInterfaceList() { - cp := iface.GetCopy() - - if !iface.Equal(cp) { - t.Fatalf("Failed to return a copy of Interface") - } - - if iface == cp { - t.Fatalf("Failed to return a true copy of Interface") - } - } -} - -func TestSandboxInfoCopy(t *testing.T) { - si := Info{Interfaces: getInterfaceList(), Gateway: net.ParseIP("192.168.1.254"), GatewayIPv6: net.ParseIP("2001:2345::abcd:8889")} - cp := si.GetCopy() - - if !si.Equal(cp) { - t.Fatalf("Failed to return a copy of Info") - } - - if &si == cp { - t.Fatalf("Failed to return a true copy of Info") - } -} - -func getInterfaceList() []*Interface { - _, netv4a, _ := net.ParseCIDR("192.168.30.1/24") - _, netv4b, _ := net.ParseCIDR("172.18.255.2/23") - _, netv6a, _ := net.ParseCIDR("2001:2345::abcd:8888/80") - _, netv6b, _ := net.ParseCIDR("2001:2345::abcd:8889/80") - - return []*Interface{ - &Interface{ - SrcName: "veth1234567", - DstName: "eth0", - Address: netv4a, - AddressIPv6: netv6a, - Routes: []*net.IPNet{netv4a, netv6a}, - }, - &Interface{ - SrcName: "veth7654321", - DstName: "eth1", - Address: netv4b, - AddressIPv6: netv6b, - Routes: []*net.IPNet{netv4b, netv6b}, - }, - } -} diff --git a/sandboxdata.go b/sandboxdata.go index 947fea3..58e6986 100644 --- a/sandboxdata.go +++ b/sandboxdata.go @@ -83,17 +83,16 @@ func (s *sandboxData) addEndpoint(ep *endpoint) error { sb := s.sandbox() for _, i := range ifaces { - iface := &sandbox.Interface{ - SrcName: i.srcName, - DstName: i.dstPrefix, - Address: &i.addr, - Routes: i.routes, - } + var ifaceOptions []sandbox.IfaceOption + + ifaceOptions = append(ifaceOptions, sb.InterfaceOptions().Address(&i.addr), + sb.InterfaceOptions().Routes(i.routes)) if i.addrv6.IP.To16() != nil { - iface.AddressIPv6 = &i.addrv6 + ifaceOptions = append(ifaceOptions, + sb.InterfaceOptions().AddressIPv6(&i.addrv6)) } - if err := sb.AddInterface(iface); err != nil { + if err := sb.AddInterface(i.srcName, i.dstPrefix, ifaceOptions...); err != nil { return err } } @@ -131,10 +130,10 @@ func (s *sandboxData) rmEndpoint(ep *endpoint) int { ep.Unlock() sb := s.sandbox() - for _, i := range sb.Interfaces() { + for _, i := range sb.Info().Interfaces() { // Only remove the interfaces owned by this endpoint from the sandbox. - if ep.hasInterface(i.SrcName) { - if err := sb.RemoveInterface(i); err != nil { + if ep.hasInterface(i.SrcName()) { + if err := i.Remove(); err != nil { logrus.Debugf("Remove interface failed: %v", err) } } From 4aa51271fedf713b2d91f1e7c3243492a9d562ad Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Thu, 4 Jun 2015 23:45:04 -0700 Subject: [PATCH 3/3] Add support to add bridge to the sandbox. Added support to add a bridge the same way as any other interface into the namespace. The only difference is linux does not support creating the bridge in one namespace and moving it into another namespace. So for a bridge the sandbox code also does the creation of the bridge inside the sandbox. Also added an optional argument to interface which can now select one of the already existing interfaces as it's master. For this option to succeed the master interface should be of type bridge. Signed-off-by: Jana Radhakrishnan --- sandbox/interface_linux.go | 29 +++++++++++++++++++++++++++++ sandbox/options_linux.go | 12 ++++++++++++ sandbox/sandbox.go | 14 ++++++++++++++ sandbox/sandbox_linux_test.go | 30 ++++++++++-------------------- sandbox/sandbox_test.go | 11 +++++++---- 5 files changed, 72 insertions(+), 24 deletions(-) diff --git a/sandbox/interface_linux.go b/sandbox/interface_linux.go index 0933985..73bd1af 100644 --- a/sandbox/interface_linux.go +++ b/sandbox/interface_linux.go @@ -15,9 +15,12 @@ type IfaceOption func(i *nwIface) type nwIface struct { srcName string dstName string + master string + dstMaster string address *net.IPNet addressIPv6 *net.IPNet routes []*net.IPNet + bridge bool ns *networkNamespace sync.Mutex } @@ -169,6 +172,14 @@ func (n *networkNamespace) AddInterface(srcName, dstPrefix string, options ...If i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n} i.processInterfaceOptions(options...) + if i.master != "" { + i.dstMaster = n.findDstMaster(i.master) + if i.dstMaster == "" { + return fmt.Errorf("could not find an appropriate master %q for %q", + i.master, i.srcName) + } + } + n.Lock() i.dstName = fmt.Sprintf("%s%d", i.dstName, n.nextIfIndex) n.nextIfIndex++ @@ -176,6 +187,12 @@ func (n *networkNamespace) AddInterface(srcName, dstPrefix string, options ...If n.Unlock() return nsInvoke(path, func(nsFD int) error { + // If it is a bridge interface we have to create the bridge inside + // the namespace so don't try to lookup the interface using srcName + if i.bridge { + return nil + } + // Find the network interface identified by the SrcName attribute. iface, err := netlink.LinkByName(i.srcName) if err != nil { @@ -189,6 +206,18 @@ func (n *networkNamespace) AddInterface(srcName, dstPrefix string, options ...If return nil }, func(callerFD int) error { + if i.bridge { + link := &netlink.Bridge{ + LinkAttrs: netlink.LinkAttrs{ + Name: i.srcName, + }, + } + + if err := netlink.LinkAdd(link); err != nil { + return fmt.Errorf("failed to create bridge %q: %v", i.srcName, err) + } + } + // Find the network interface identified by the SrcName attribute. iface, err := netlink.LinkByName(i.srcName) if err != nil { diff --git a/sandbox/options_linux.go b/sandbox/options_linux.go index 40199e0..4064848 100644 --- a/sandbox/options_linux.go +++ b/sandbox/options_linux.go @@ -10,6 +10,18 @@ func (i *nwIface) processInterfaceOptions(options ...IfaceOption) { } } +func (n *networkNamespace) Bridge(isBridge bool) IfaceOption { + return func(i *nwIface) { + i.bridge = isBridge + } +} + +func (n *networkNamespace) Master(name string) IfaceOption { + return func(i *nwIface) { + i.master = name + } +} + func (n *networkNamespace) Address(addr *net.IPNet) IfaceOption { return func(i *nwIface) { i.address = addr diff --git a/sandbox/sandbox.go b/sandbox/sandbox.go index 92bf4cc..f08e074 100644 --- a/sandbox/sandbox.go +++ b/sandbox/sandbox.go @@ -49,12 +49,20 @@ type Sandbox interface { // IfaceOptionSetter interface defines the option setter methods for interface options. type IfaceOptionSetter interface { + // Bridge returns an option setter to set if the interface is a bridge. + Bridge(bool) IfaceOption + // Address returns an option setter to set IPv4 address. Address(*net.IPNet) IfaceOption // Address returns an option setter to set IPv6 address. AddressIPv6(*net.IPNet) IfaceOption + // Master returns an option setter to set the master interface if any for this + // interface. The master interface name should refer to the srcname of a + // previously added interface of type bridge. + Master(string) IfaceOption + // Address returns an option setter to set interface routes. Routes([]*net.IPNet) IfaceOption } @@ -106,6 +114,12 @@ type Interface interface { // IP routes for the interface. Routes() []*net.IPNet + // Bridge returns true if the interface is a bridge + Bridge() bool + + // Master returns the srcname of the master interface for this interface. + Master() string + // Remove an interface from the sandbox by renaming to original name // and moving it out of the sandbox. Remove() error diff --git a/sandbox/sandbox_linux_test.go b/sandbox/sandbox_linux_test.go index e47ea78..7f769c9 100644 --- a/sandbox/sandbox_linux_test.go +++ b/sandbox/sandbox_linux_test.go @@ -76,6 +76,11 @@ func newInfo(t *testing.T) (Sandbox, error) { intf1.routes = []*net.IPNet{route} + intf2 := &nwIface{} + intf2.srcName = "testbridge" + intf2.dstName = sboxIfaceName + intf2.bridge = true + veth = &netlink.Veth{ LinkAttrs: netlink.LinkAttrs{Name: vethName3, TxQLen: 0}, PeerName: vethName4} @@ -84,27 +89,12 @@ func newInfo(t *testing.T) (Sandbox, error) { return nil, err } - intf2 := &nwIface{} - intf2.srcName = vethName4 - intf2.dstName = sboxIfaceName + intf3 := &nwIface{} + intf3.srcName = vethName4 + intf3.dstName = sboxIfaceName + intf3.master = "testbridge" - ip4, addr, err = net.ParseCIDR("192.168.2.100/24") - if err != nil { - return nil, err - } - intf2.address = addr - intf2.address.IP = ip4 - - // ip6, addrv6, err := net.ParseCIDR("2001:DB8::ABCD/48") - ip6, addrv6, err = net.ParseCIDR("fe80::3/64") - - if err != nil { - return nil, err - } - intf2.addressIPv6 = addrv6 - intf2.addressIPv6.IP = ip6 - - info := &networkNamespace{iFaces: []*nwIface{intf1, intf2}} + info := &networkNamespace{iFaces: []*nwIface{intf1, intf2, intf3}} info.gw = net.ParseIP("192.168.1.1") // sinfo.GatewayIPv6 = net.ParseIP("2001:DB8::1") diff --git a/sandbox/sandbox_test.go b/sandbox/sandbox_test.go index 19e87ca..6dd4d5e 100644 --- a/sandbox/sandbox_test.go +++ b/sandbox/sandbox_test.go @@ -41,6 +41,7 @@ func TestSandboxCreate(t *testing.T) { for _, i := range tbox.Info().Interfaces() { err = s.AddInterface(i.SrcName(), i.DstName(), + tbox.InterfaceOptions().Bridge(i.Bridge()), tbox.InterfaceOptions().Address(i.Address()), tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6())) if err != nil { @@ -61,7 +62,7 @@ func TestSandboxCreate(t *testing.T) { } runtime.LockOSThread() - verifySandbox(t, s, []string{"0", "1"}) + verifySandbox(t, s, []string{"0", "1", "2"}) runtime.LockOSThread() s.Destroy() @@ -134,6 +135,7 @@ func TestAddRemoveInterface(t *testing.T) { for _, i := range tbox.Info().Interfaces() { err = s.AddInterface(i.SrcName(), i.DstName(), + tbox.InterfaceOptions().Bridge(i.Bridge()), tbox.InterfaceOptions().Address(i.Address()), tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6())) if err != nil { @@ -142,7 +144,7 @@ func TestAddRemoveInterface(t *testing.T) { runtime.LockOSThread() } - verifySandbox(t, s, []string{"0", "1"}) + verifySandbox(t, s, []string{"0", "1", "2"}) runtime.LockOSThread() interfaces := s.Info().Interfaces() @@ -151,18 +153,19 @@ func TestAddRemoveInterface(t *testing.T) { } runtime.LockOSThread() - verifySandbox(t, s, []string{"1"}) + verifySandbox(t, s, []string{"1", "2"}) runtime.LockOSThread() i := tbox.Info().Interfaces()[0] if err := s.AddInterface(i.SrcName(), i.DstName(), + tbox.InterfaceOptions().Bridge(i.Bridge()), tbox.InterfaceOptions().Address(i.Address()), tbox.InterfaceOptions().AddressIPv6(i.AddressIPv6())); err != nil { t.Fatalf("Failed to add interfaces to sandbox: %v", err) } runtime.LockOSThread() - verifySandbox(t, s, []string{"1", "2"}) + verifySandbox(t, s, []string{"1", "2", "3"}) runtime.LockOSThread() s.Destroy()