From 845ef40a4560929eaa9ca5022616120d6ff213d5 Mon Sep 17 00:00:00 2001 From: Jana Radhakrishnan Date: Thu, 8 Oct 2015 14:59:47 -0700 Subject: [PATCH] Make bridge driver networks persistent Since libnetwork is going to provide createNetwork notifications only once when the network is created bridge network needs to save it's network state in persistent store so that it becomes available even after restart. Signed-off-by: Jana Radhakrishnan --- datastore/datastore.go | 4 +- datastore/datastore_test.go | 1 - drivers/bridge/bridge.go | 49 +++++--- drivers/bridge/bridge_store.go | 210 +++++++++++++++++++++++++++++++++ drivers/bridge/bridge_test.go | 8 -- ipam/allocator_test.go | 1 - network.go | 22 ---- 7 files changed, 245 insertions(+), 50 deletions(-) create mode 100644 drivers/bridge/bridge_store.go diff --git a/datastore/datastore.go b/datastore/datastore.go index 3c63949..3514dc6 100644 --- a/datastore/datastore.go +++ b/datastore/datastore.go @@ -92,8 +92,7 @@ type KVConstructor interface { // ScopeCfg represents Datastore configuration. type ScopeCfg struct { - Embedded bool - Client ScopeClientCfg + Client ScopeClientCfg } // ScopeClientCfg represents Datastore Client-only mode configuration @@ -125,7 +124,6 @@ var ( func makeDefaultScopes() map[string]*ScopeCfg { def := make(map[string]*ScopeCfg) def[LocalScope] = &ScopeCfg{ - Embedded: true, Client: ScopeClientCfg{ Provider: "boltdb", Address: defaultPrefix + "/boltdb.db", diff --git a/datastore/datastore_test.go b/datastore/datastore_test.go index 4e93abd..145f03c 100644 --- a/datastore/datastore_test.go +++ b/datastore/datastore_test.go @@ -38,7 +38,6 @@ func TestParseKey(t *testing.T) { func TestInvalidDataStore(t *testing.T) { config := &ScopeCfg{} - config.Embedded = false config.Client.Provider = "invalid" config.Client.Address = "localhost:8500" _, err := NewDataStore(GlobalScope, config) diff --git a/drivers/bridge/bridge.go b/drivers/bridge/bridge.go index 040de05..600ff74 100644 --- a/drivers/bridge/bridge.go +++ b/drivers/bridge/bridge.go @@ -55,6 +55,7 @@ type configuration struct { // networkConfiguration for network specific configuration type networkConfiguration struct { + ID string BridgeName string EnableIPv6 bool EnableIPMasquerade bool @@ -67,6 +68,8 @@ type networkConfiguration struct { AddressIPv6 *net.IPNet DefaultGatewayIPv4 net.IP DefaultGatewayIPv6 net.IP + dbIndex uint64 + dbExists bool } // endpointConfiguration represents the user specified configuration for the sandbox endpoint @@ -109,6 +112,7 @@ type driver struct { natChain *iptables.ChainInfo filterChain *iptables.ChainInfo networks map[string]*bridgeNetwork + store datastore.DataStore sync.Mutex } @@ -376,6 +380,11 @@ func (d *driver) configure(option map[string]interface{}) error { var config *configuration var err error + err = d.initStore(option) + if err != nil { + return err + } + d.Lock() defer d.Unlock() @@ -511,6 +520,8 @@ func parseNetworkOptions(id string, option options.Generic) (*networkConfigurati if config.BridgeName == "" && config.DefaultBridge == false { config.BridgeName = "br-" + id[:12] } + + config.ID = id return config, nil } @@ -540,10 +551,6 @@ func (d *driver) getNetworks() []*bridgeNetwork { // Create a new network using bridge plugin func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) error { - var err error - - defer osl.InitOSContext()() - // Sanity checks d.Lock() if _, ok := d.networks[id]; ok { @@ -563,6 +570,18 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat return err } + if err = d.createNetwork(config); err != nil { + return err + } + + return d.storeUpdate(config) +} + +func (d *driver) createNetwork(config *networkConfiguration) error { + var err error + + defer osl.InitOSContext()() + networkList := d.getNetworks() for _, nw := range networkList { nw.Lock() @@ -570,13 +589,13 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat nw.Unlock() if err := nwConfig.Conflicts(config); err != nil { return types.ForbiddenErrorf("cannot create network %s (%s): conflicts with network %s (%s): %s", - nwConfig.BridgeName, id, nw.id, nw.config.BridgeName, err.Error()) + nwConfig.BridgeName, config.ID, nw.id, nw.config.BridgeName, err.Error()) } } // Create and set network handler in driver network := &bridgeNetwork{ - id: id, + id: config.ID, endpoints: make(map[string]*bridgeEndpoint), config: config, portMapper: portmapper.New(), @@ -584,14 +603,14 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat } d.Lock() - d.networks[id] = network + d.networks[config.ID] = network d.Unlock() // On failure make sure to reset driver network handler to nil defer func() { if err != nil { d.Lock() - delete(d.networks, id) + delete(d.networks, config.ID) d.Unlock() } }() @@ -604,7 +623,7 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat // networks. This step is needed now because driver might have now set the bridge // name on this config struct. And because we need to check for possible address // conflicts, so we need to check against operationa lnetworks. - if err = config.conflictsWithNetworks(id, networkList); err != nil { + if err = config.conflictsWithNetworks(config.ID, networkList); err != nil { return err } @@ -705,10 +724,6 @@ func (d *driver) DeleteNetwork(nid string) error { config := n.config n.Unlock() - if config.BridgeName == DefaultBridgeName { - return types.ForbiddenErrorf("default network of type \"%s\" cannot be deleted", networkType) - } - d.Lock() delete(d.networks, nid) d.Unlock() @@ -753,8 +768,12 @@ func (d *driver) DeleteNetwork(nid string) error { return err } - // Programming - return netlink.LinkDel(n.bridge.Link) + // We only delete the bridge when it's not the default bridge. This is keep the backward compatible behavior. + if !config.DefaultBridge { + err = netlink.LinkDel(n.bridge.Link) + } + + return d.storeDelete(config) } func addToBridge(ifaceName, bridgeName string) error { diff --git a/drivers/bridge/bridge_store.go b/drivers/bridge/bridge_store.go new file mode 100644 index 0000000..bea172c --- /dev/null +++ b/drivers/bridge/bridge_store.go @@ -0,0 +1,210 @@ +package bridge + +import ( + "encoding/json" + "fmt" + "net" + + "github.com/Sirupsen/logrus" + "github.com/docker/libkv/store" + "github.com/docker/libkv/store/boltdb" + "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/types" +) + +const bridgePrefix = "bridge" + +func (d *driver) initStore(option map[string]interface{}) error { + var err error + + provider, provOk := option[netlabel.LocalKVProvider] + provURL, urlOk := option[netlabel.LocalKVProviderURL] + + if provOk && urlOk { + cfg := &datastore.ScopeCfg{ + Client: datastore.ScopeClientCfg{ + Provider: provider.(string), + Address: provURL.(string), + }, + } + + provConfig, confOk := option[netlabel.LocalKVProviderConfig] + if confOk { + cfg.Client.Config = provConfig.(*store.Config) + } + + d.store, err = datastore.NewDataStore(datastore.LocalScope, cfg) + if err != nil { + return fmt.Errorf("bridge driver failed to initialize data store: %v", err) + } + + return d.populateNetworks() + } + + return nil +} + +func (d *driver) populateNetworks() error { + kvol, err := d.store.List(datastore.Key(bridgePrefix), &networkConfiguration{}) + if err != nil && err != datastore.ErrKeyNotFound && err != boltdb.ErrBoltBucketNotFound { + return fmt.Errorf("failed to get bridge network configurations from store: %v", err) + } + + // It's normal for network configuration state to be empty. Just return. + if err == datastore.ErrKeyNotFound { + return nil + } + + for _, kvo := range kvol { + ncfg := kvo.(*networkConfiguration) + if err = d.createNetwork(ncfg); err != nil { + logrus.Warnf("could not create bridge network for id %s bridge name %s while booting up from persistent state", ncfg.ID, ncfg.BridgeName) + } + } + + return nil +} + +func (d *driver) storeUpdate(kvObject datastore.KVObject) error { + if d.store == nil { + logrus.Warnf("bridge store not initialized. kv object %s is not added to the store", datastore.Key(kvObject.Key()...)) + return nil + } + + if err := d.store.PutObjectAtomic(kvObject); err != nil { + return fmt.Errorf("failed to update bridge store for object type %T: %v", kvObject, err) + } + + return nil +} + +func (d *driver) storeDelete(kvObject datastore.KVObject) error { + if d.store == nil { + logrus.Debugf("bridge store not initialized. kv object %s is not deleted from store", datastore.Key(kvObject.Key()...)) + return nil + } + +retry: + if err := d.store.DeleteObjectAtomic(kvObject); err != nil { + if err == datastore.ErrKeyModified { + if err := d.store.GetObject(datastore.Key(kvObject.Key()...), kvObject); err != nil { + return fmt.Errorf("could not update the kvobject to latest when trying to delete: %v", err) + } + goto retry + } + return err + } + + return nil +} + +func (ncfg *networkConfiguration) MarshalJSON() ([]byte, error) { + nMap := make(map[string]interface{}) + nMap["ID"] = ncfg.ID + nMap["BridgeName"] = ncfg.BridgeName + nMap["EnableIPv6"] = ncfg.EnableIPv6 + nMap["EnableIPMasquerade"] = ncfg.EnableIPMasquerade + nMap["EnableICC"] = ncfg.EnableICC + nMap["Mtu"] = ncfg.Mtu + nMap["DefaultBridge"] = ncfg.DefaultBridge + nMap["DefaultBindingIP"] = ncfg.DefaultBindingIP.String() + nMap["DefaultGatewayIPv4"] = ncfg.DefaultGatewayIPv4.String() + nMap["DefaultGatewayIPv6"] = ncfg.DefaultGatewayIPv6.String() + + if ncfg.AddressIPv4 != nil { + nMap["AddressIPv4"] = ncfg.AddressIPv4.String() + } + + if ncfg.AddressIPv6 != nil { + nMap["AddressIPv6"] = ncfg.AddressIPv6.String() + } + + return json.Marshal(nMap) +} + +func (ncfg *networkConfiguration) UnmarshalJSON(b []byte) error { + var ( + err error + nMap map[string]interface{} + ) + + if err = json.Unmarshal(b, &nMap); err != nil { + return err + } + + if v, ok := nMap["AddressIPv4"]; ok { + if ncfg.AddressIPv4, err = types.ParseCIDR(v.(string)); err != nil { + return types.InternalErrorf("failed to decode bridge network address IPv4 after json unmarshal: %s", v.(string)) + } + } + + if v, ok := nMap["AddressIPv6"]; ok { + if ncfg.AddressIPv6, err = types.ParseCIDR(v.(string)); err != nil { + return types.InternalErrorf("failed to decode bridge network address IPv6 after json unmarshal: %s", v.(string)) + } + } + + ncfg.DefaultBindingIP = net.ParseIP(nMap["DefaultBindingIP"].(string)) + ncfg.DefaultGatewayIPv4 = net.ParseIP(nMap["DefaultGatewayIPv4"].(string)) + ncfg.DefaultGatewayIPv6 = net.ParseIP(nMap["DefaultGatewayIPv6"].(string)) + ncfg.ID = nMap["ID"].(string) + ncfg.BridgeName = nMap["BridgeName"].(string) + ncfg.EnableIPv6 = nMap["EnableIPv6"].(bool) + ncfg.EnableIPMasquerade = nMap["EnableIPMasquerade"].(bool) + ncfg.EnableICC = nMap["EnableICC"].(bool) + ncfg.Mtu = int(nMap["Mtu"].(float64)) + + return nil +} + +func (ncfg *networkConfiguration) Key() []string { + return []string{bridgePrefix, ncfg.ID} +} + +func (ncfg *networkConfiguration) KeyPrefix() []string { + return []string{bridgePrefix} +} + +func (ncfg *networkConfiguration) Value() []byte { + b, err := json.Marshal(ncfg) + if err != nil { + return nil + } + return b +} + +func (ncfg *networkConfiguration) SetValue(value []byte) error { + return json.Unmarshal(value, ncfg) +} + +func (ncfg *networkConfiguration) Index() uint64 { + return ncfg.dbIndex +} + +func (ncfg *networkConfiguration) SetIndex(index uint64) { + ncfg.dbIndex = index + ncfg.dbExists = true +} + +func (ncfg *networkConfiguration) Exists() bool { + return ncfg.dbExists +} + +func (ncfg *networkConfiguration) Skip() bool { + return false +} + +func (ncfg *networkConfiguration) New() datastore.KVObject { + return &networkConfiguration{} +} + +func (ncfg *networkConfiguration) CopyTo(o datastore.KVObject) error { + dstNcfg := o.(*networkConfiguration) + *dstNcfg = *ncfg + return nil +} + +func (ncfg *networkConfiguration) DataScope() string { + return datastore.LocalScope +} diff --git a/drivers/bridge/bridge_test.go b/drivers/bridge/bridge_test.go index 40ce65d..e1891af 100644 --- a/drivers/bridge/bridge_test.go +++ b/drivers/bridge/bridge_test.go @@ -120,14 +120,6 @@ func TestCreate(t *testing.T) { if _, ok := err.(types.ForbiddenError); !ok { t.Fatalf("Creation of second network with default name failed with unexpected error type") } - - err = d.DeleteNetwork("dummy") - if err == nil { - t.Fatalf("deletion of network with default name should fail on this driver") - } - if _, ok := err.(types.ForbiddenError); !ok { - t.Fatalf("deletion of network with default name failed with unexpected error type") - } } func TestCreateFail(t *testing.T) { diff --git a/ipam/allocator_test.go b/ipam/allocator_test.go index df3b01f..e38d08c 100644 --- a/ipam/allocator_test.go +++ b/ipam/allocator_test.go @@ -32,7 +32,6 @@ func randomLocalStore() (datastore.DataStore, error) { return nil, fmt.Errorf("Error closing temp file: %v", err) } return datastore.NewDataStore(datastore.LocalScope, &datastore.ScopeCfg{ - Embedded: true, Client: datastore.ScopeClientCfg{ Provider: "boltdb", Address: defaultPrefix + tmp.Name(), diff --git a/network.go b/network.go index c58dd82..4537273 100644 --- a/network.go +++ b/network.go @@ -557,17 +557,6 @@ func (n *network) deleteNetwork() error { return fmt.Errorf("failed deleting network: %v", err) } - // If it is bridge network type make sure we call the driver about the network - // because the network may have been created in some past life of libnetwork. - if n.Type() == "bridge" { - n.drvOnce.Do(func() { - err = n.getController().addNetwork(n) - }) - if err != nil { - return err - } - } - if err := d.DeleteNetwork(n.ID()); err != nil { // Forbidden Errors should be honored if _, ok := err.(types.ForbiddenError); ok { @@ -585,17 +574,6 @@ func (n *network) addEndpoint(ep *endpoint) error { return fmt.Errorf("failed to add endpoint: %v", err) } - // If it is bridge network type make sure we call the driver about the network - // because the network may have been created in some past life of libnetwork. - if n.Type() == "bridge" { - n.drvOnce.Do(func() { - err = n.getController().addNetwork(n) - }) - if err != nil { - return err - } - } - err = d.CreateEndpoint(n.id, ep.id, ep.Interface(), ep.generic) if err != nil { return types.InternalErrorf("failed to create endpoint %s on network %s: %v",