diff --git a/drivers/bridge/bridge.go b/drivers/bridge/bridge.go index ceff412..3cacad1 100644 --- a/drivers/bridge/bridge.go +++ b/drivers/bridge/bridge.go @@ -3,10 +3,12 @@ package bridge import ( "errors" "net" + "os/exec" "strconv" "strings" "sync" + "github.com/Sirupsen/logrus" "github.com/docker/libnetwork/driverapi" "github.com/docker/libnetwork/ipallocator" "github.com/docker/libnetwork/netlabel" @@ -104,6 +106,12 @@ func newDriver() driverapi.Driver { // Init registers a new instance of bridge driver func Init(dc driverapi.DriverCallback) error { + // try to modprobe bridge first + // see gh#12177 + if out, err := exec.Command("modprobe", "-va", "bridge", "nf_nat", "br_netfilter").Output(); err != nil { + logrus.Warnf("Running modprobe bridge nf_nat failed with message: %s, error: %v", out, err) + } + return dc.RegisterDriver(networkType, newDriver()) } @@ -510,6 +518,11 @@ func (d *driver) CreateNetwork(id types.UUID, option map[string]interface{}) err // Even if a bridge exists try to setup IPv4. bridgeSetup.queueStep(setupBridgeIPv4) + enableIPv6Forwarding := false + if d.config != nil && d.config.EnableIPForwarding && config.FixedCIDRv6 != nil { + enableIPv6Forwarding = true + } + // Conditionally queue setup steps depending on configuration values. for _, step := range []struct { Condition bool @@ -533,6 +546,9 @@ func (d *driver) CreateNetwork(id types.UUID, option map[string]interface{}) err // specified subnet. {config.FixedCIDRv6 != nil, setupFixedCIDRv6}, + // Enable IPv6 Forwarding + {enableIPv6Forwarding, setupIPv6Forwarding}, + // Setup Loopback Adresses Routing {!config.EnableUserlandProxy, setupLoopbackAdressesRouting}, diff --git a/drivers/bridge/setup_fixedcidrv6.go b/drivers/bridge/setup_fixedcidrv6.go index 1b4bb57..b2a949b 100644 --- a/drivers/bridge/setup_fixedcidrv6.go +++ b/drivers/bridge/setup_fixedcidrv6.go @@ -1,7 +1,10 @@ package bridge import ( + "os" + log "github.com/Sirupsen/logrus" + "github.com/vishvananda/netlink" ) func setupFixedCIDRv6(config *networkConfiguration, i *bridgeInterface) error { @@ -10,5 +13,15 @@ func setupFixedCIDRv6(config *networkConfiguration, i *bridgeInterface) error { return &FixedCIDRv6Error{Net: config.FixedCIDRv6, Err: err} } + // Setting route to global IPv6 subnet + log.Debugf("Adding route to IPv6 network %s via device %s", config.FixedCIDRv6.String(), config.BridgeName) + err := netlink.RouteAdd(&netlink.Route{ + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: i.Link.Attrs().Index, + Dst: config.FixedCIDRv6, + }) + if err != nil && !os.IsExist(err) { + log.Errorf("Could not add route to IPv6 network %s via device %s", config.FixedCIDRv6.String(), config.BridgeName) + } return nil } diff --git a/drivers/bridge/setup_ipv6.go b/drivers/bridge/setup_ipv6.go index b797af8..b534644 100644 --- a/drivers/bridge/setup_ipv6.go +++ b/drivers/bridge/setup_ipv6.go @@ -5,12 +5,16 @@ import ( "io/ioutil" "net" + "github.com/Sirupsen/logrus" "github.com/vishvananda/netlink" ) var bridgeIPv6 *net.IPNet -const bridgeIPv6Str = "fe80::1/64" +const ( + bridgeIPv6Str = "fe80::1/64" + ipv6ForwardConfPerm = 0644 +) func init() { // We allow ourselves to panic in this special case because we indicate a @@ -25,7 +29,7 @@ func init() { func setupBridgeIPv6(config *networkConfiguration, i *bridgeInterface) error { // Enable IPv6 on the bridge procFile := "/proc/sys/net/ipv6/conf/" + config.BridgeName + "/disable_ipv6" - if err := ioutil.WriteFile(procFile, []byte{'0', '\n'}, 0644); err != nil { + if err := ioutil.WriteFile(procFile, []byte{'0', '\n'}, ipv6ForwardConfPerm); err != nil { return fmt.Errorf("Unable to enable IPv6 addresses on bridge: %v", err) } @@ -64,3 +68,14 @@ func setupGatewayIPv6(config *networkConfiguration, i *bridgeInterface) error { return nil } + +func setupIPv6Forwarding(config *networkConfiguration, i *bridgeInterface) error { + // Enable IPv6 forwarding + if err := ioutil.WriteFile("/proc/sys/net/ipv6/conf/default/forwarding", []byte{'1', '\n'}, ipv6ForwardConfPerm); err != nil { + logrus.Warnf("Unable to enable IPv6 default forwarding: %v", err) + } + if err := ioutil.WriteFile("/proc/sys/net/ipv6/conf/all/forwarding", []byte{'1', '\n'}, ipv6ForwardConfPerm); err != nil { + logrus.Warnf("Unable to enable IPv6 all forwarding: %v", err) + } + return nil +} diff --git a/network.go b/network.go index 7b15ffc..8a25b84 100644 --- a/network.go +++ b/network.go @@ -64,14 +64,23 @@ type network struct { } func (n *network) Name() string { + n.Lock() + defer n.Unlock() + return n.name } func (n *network) ID() string { + n.Lock() + defer n.Unlock() + return string(n.id) } func (n *network) Type() string { + n.Lock() + defer n.Unlock() + if n.driver == nil { return "" } diff --git a/sandbox/namespace_linux.go b/sandbox/namespace_linux.go index dd93e83..3db3a8e 100644 --- a/sandbox/namespace_linux.go +++ b/sandbox/namespace_linux.go @@ -4,10 +4,14 @@ import ( "fmt" "net" "os" + "os/exec" "runtime" "sync" "syscall" + "time" + log "github.com/Sirupsen/logrus" + "github.com/docker/docker/pkg/reexec" "github.com/docker/libnetwork/types" "github.com/vishvananda/netlink" "github.com/vishvananda/netns" @@ -15,7 +19,13 @@ import ( const prefix = "/var/run/docker/netns" -var once sync.Once +var ( + once sync.Once + garbagePathMap = make(map[string]bool) + gpmLock sync.Mutex + gpmWg sync.WaitGroup + gpmCleanupPeriod = 60 * time.Second +) // The networkNamespace type is the linux implementation of the Sandbox // interface. It represents a linux network namespace, and moves an interface @@ -27,11 +37,53 @@ type networkNamespace struct { sync.Mutex } +func init() { + reexec.Register("netns-create", reexecCreateNamespace) +} + func createBasePath() { err := os.MkdirAll(prefix, 0644) if err != nil && !os.IsExist(err) { panic("Could not create net namespace path directory") } + + // Start the garbage collection go routine + go removeUnusedPaths() +} + +func removeUnusedPaths() { + gpmLock.Lock() + period := gpmCleanupPeriod + gpmLock.Unlock() + + for range time.Tick(period) { + gpmLock.Lock() + pathList := make([]string, 0, len(garbagePathMap)) + for path := range garbagePathMap { + pathList = append(pathList, path) + } + garbagePathMap = make(map[string]bool) + gpmWg.Add(1) + gpmLock.Unlock() + + for _, path := range pathList { + os.Remove(path) + } + + gpmWg.Done() + } +} + +func addToGarbagePaths(path string) { + gpmLock.Lock() + garbagePathMap[path] = true + gpmLock.Unlock() +} + +func removeFromGarbagePaths(path string) { + gpmLock.Lock() + delete(garbagePathMap, path) + gpmLock.Unlock() } // GenerateKey generates a sandbox key based on the passed @@ -56,6 +108,20 @@ func NewSandbox(key string, osCreate bool) (Sandbox, error) { return &networkNamespace{path: key, sinfo: info}, nil } +func reexecCreateNamespace() { + if len(os.Args) < 2 { + log.Fatal("no namespace path provided") + } + + if err := syscall.Mount("/proc/self/ns/net", os.Args[1], "bind", syscall.MS_BIND, ""); err != nil { + log.Fatal(err) + } + + if err := loopbackUp(); err != nil { + log.Fatal(err) + } +} + func createNetworkNamespace(path string, osCreate bool) (*Info, error) { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -70,23 +136,18 @@ func createNetworkNamespace(path string, osCreate bool) (*Info, error) { return nil, err } - if osCreate { - defer netns.Set(origns) - newns, err := netns.New() - if err != nil { - return nil, err - } - defer newns.Close() - - if err := loopbackUp(); err != nil { - return nil, err - } + cmd := &exec.Cmd{ + Path: reexec.Self(), + Args: append([]string{"netns-create"}, path), + Stdout: os.Stdout, + Stderr: os.Stderr, } - - procNet := fmt.Sprintf("/proc/%d/task/%d/ns/net", os.Getpid(), syscall.Gettid()) - - if err := syscall.Mount(procNet, path, "bind", syscall.MS_BIND, ""); err != nil { - return nil, err + if osCreate { + cmd.SysProcAttr = &syscall.SysProcAttr{} + cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNET + } + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("namespace creation reexec command failed: %v", err) } interfaces := []*Interface{} @@ -94,10 +155,9 @@ func createNetworkNamespace(path string, osCreate bool) (*Info, error) { return info, nil } -func cleanupNamespaceFile(path string) { +func unmountNamespaceFile(path string) { if _, err := os.Stat(path); err == nil { - n := &networkNamespace{path: path} - n.Destroy() + syscall.Unmount(path, syscall.MNT_DETACH) } } @@ -105,11 +165,20 @@ func createNamespaceFile(path string) (err error) { var f *os.File once.Do(createBasePath) - // cleanup namespace file if it already exists because of a previous ungraceful exit. - cleanupNamespaceFile(path) + // Remove it from garbage collection list if present + removeFromGarbagePaths(path) + + // If the path is there unmount it first + unmountNamespaceFile(path) + + // wait for garbage collection to complete if it is in progress + // before trying to create the file. + gpmWg.Wait() + if f, err = os.Create(path); err == nil { f.Close() } + return err } @@ -310,5 +379,7 @@ func (n *networkNamespace) Destroy() error { return err } - return os.Remove(n.path) + // Stash it into the garbage collection list + addToGarbagePaths(n.path) + return nil } diff --git a/sandbox/sandbox_linux_test.go b/sandbox/sandbox_linux_test.go index 678cd0d..b00d14f 100644 --- a/sandbox/sandbox_linux_test.go +++ b/sandbox/sandbox_linux_test.go @@ -6,6 +6,7 @@ import ( "path/filepath" "runtime" "testing" + "time" "github.com/docker/libnetwork/netutils" "github.com/vishvananda/netlink" @@ -31,6 +32,11 @@ func newKey(t *testing.T) (string, error) { return "", err } + // Set the rpmCleanupPeriod to be low to make the test run quicker + gpmLock.Lock() + gpmCleanupPeriod = 2 * time.Second + gpmLock.Unlock() + return name, nil } @@ -146,3 +152,10 @@ func verifySandbox(t *testing.T, s Sandbox) { err) } } + +func verifyCleanup(t *testing.T, s Sandbox) { + time.Sleep(time.Duration(gpmCleanupPeriod * 2)) + if _, err := os.Stat(s.Key()); err == nil { + t.Fatalf("The sandbox path %s is not getting cleanup event after twice the cleanup period", s.Key()) + } +} diff --git a/sandbox/sandbox_test.go b/sandbox/sandbox_test.go index 03b250b..639dc79 100644 --- a/sandbox/sandbox_test.go +++ b/sandbox/sandbox_test.go @@ -2,9 +2,19 @@ package sandbox import ( "net" + "os" "testing" + + "github.com/docker/docker/pkg/reexec" ) +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) +} + func TestSandboxCreate(t *testing.T) { key, err := newKey(t) if err != nil { @@ -44,6 +54,7 @@ func TestSandboxCreate(t *testing.T) { verifySandbox(t, s) s.Destroy() + verifyCleanup(t, s) } func TestSandboxCreateTwice(t *testing.T) {