diff --git a/datastore/datastore.go b/datastore/datastore.go index 3c63949..3514dc6 100644 --- a/datastore/datastore.go +++ b/datastore/datastore.go @@ -92,8 +92,7 @@ type KVConstructor interface { // ScopeCfg represents Datastore configuration. type ScopeCfg struct { - Embedded bool - Client ScopeClientCfg + Client ScopeClientCfg } // ScopeClientCfg represents Datastore Client-only mode configuration @@ -125,7 +124,6 @@ var ( func makeDefaultScopes() map[string]*ScopeCfg { def := make(map[string]*ScopeCfg) def[LocalScope] = &ScopeCfg{ - Embedded: true, Client: ScopeClientCfg{ Provider: "boltdb", Address: defaultPrefix + "/boltdb.db", diff --git a/datastore/datastore_test.go b/datastore/datastore_test.go index 4e93abd..145f03c 100644 --- a/datastore/datastore_test.go +++ b/datastore/datastore_test.go @@ -38,7 +38,6 @@ func TestParseKey(t *testing.T) { func TestInvalidDataStore(t *testing.T) { config := &ScopeCfg{} - config.Embedded = false config.Client.Provider = "invalid" config.Client.Address = "localhost:8500" _, err := NewDataStore(GlobalScope, config) diff --git a/drivers/bridge/bridge.go b/drivers/bridge/bridge.go index 040de05..600ff74 100644 --- a/drivers/bridge/bridge.go +++ b/drivers/bridge/bridge.go @@ -55,6 +55,7 @@ type configuration struct { // networkConfiguration for network specific configuration type networkConfiguration struct { + ID string BridgeName string EnableIPv6 bool EnableIPMasquerade bool @@ -67,6 +68,8 @@ type networkConfiguration struct { AddressIPv6 *net.IPNet DefaultGatewayIPv4 net.IP DefaultGatewayIPv6 net.IP + dbIndex uint64 + dbExists bool } // endpointConfiguration represents the user specified configuration for the sandbox endpoint @@ -109,6 +112,7 @@ type driver struct { natChain *iptables.ChainInfo filterChain *iptables.ChainInfo networks map[string]*bridgeNetwork + store datastore.DataStore sync.Mutex } @@ -376,6 +380,11 @@ func (d *driver) configure(option map[string]interface{}) error { var config *configuration var err error + err = d.initStore(option) + if err != nil { + return err + } + d.Lock() defer d.Unlock() @@ -511,6 +520,8 @@ func parseNetworkOptions(id string, option options.Generic) (*networkConfigurati if config.BridgeName == "" && config.DefaultBridge == false { config.BridgeName = "br-" + id[:12] } + + config.ID = id return config, nil } @@ -540,10 +551,6 @@ func (d *driver) getNetworks() []*bridgeNetwork { // Create a new network using bridge plugin func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Data, ipV6Data []driverapi.IPAMData) error { - var err error - - defer osl.InitOSContext()() - // Sanity checks d.Lock() if _, ok := d.networks[id]; ok { @@ -563,6 +570,18 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat return err } + if err = d.createNetwork(config); err != nil { + return err + } + + return d.storeUpdate(config) +} + +func (d *driver) createNetwork(config *networkConfiguration) error { + var err error + + defer osl.InitOSContext()() + networkList := d.getNetworks() for _, nw := range networkList { nw.Lock() @@ -570,13 +589,13 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat nw.Unlock() if err := nwConfig.Conflicts(config); err != nil { return types.ForbiddenErrorf("cannot create network %s (%s): conflicts with network %s (%s): %s", - nwConfig.BridgeName, id, nw.id, nw.config.BridgeName, err.Error()) + nwConfig.BridgeName, config.ID, nw.id, nw.config.BridgeName, err.Error()) } } // Create and set network handler in driver network := &bridgeNetwork{ - id: id, + id: config.ID, endpoints: make(map[string]*bridgeEndpoint), config: config, portMapper: portmapper.New(), @@ -584,14 +603,14 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat } d.Lock() - d.networks[id] = network + d.networks[config.ID] = network d.Unlock() // On failure make sure to reset driver network handler to nil defer func() { if err != nil { d.Lock() - delete(d.networks, id) + delete(d.networks, config.ID) d.Unlock() } }() @@ -604,7 +623,7 @@ func (d *driver) CreateNetwork(id string, option map[string]interface{}, ipV4Dat // networks. This step is needed now because driver might have now set the bridge // name on this config struct. And because we need to check for possible address // conflicts, so we need to check against operationa lnetworks. - if err = config.conflictsWithNetworks(id, networkList); err != nil { + if err = config.conflictsWithNetworks(config.ID, networkList); err != nil { return err } @@ -705,10 +724,6 @@ func (d *driver) DeleteNetwork(nid string) error { config := n.config n.Unlock() - if config.BridgeName == DefaultBridgeName { - return types.ForbiddenErrorf("default network of type \"%s\" cannot be deleted", networkType) - } - d.Lock() delete(d.networks, nid) d.Unlock() @@ -753,8 +768,12 @@ func (d *driver) DeleteNetwork(nid string) error { return err } - // Programming - return netlink.LinkDel(n.bridge.Link) + // We only delete the bridge when it's not the default bridge. This is keep the backward compatible behavior. + if !config.DefaultBridge { + err = netlink.LinkDel(n.bridge.Link) + } + + return d.storeDelete(config) } func addToBridge(ifaceName, bridgeName string) error { diff --git a/drivers/bridge/bridge_store.go b/drivers/bridge/bridge_store.go new file mode 100644 index 0000000..bea172c --- /dev/null +++ b/drivers/bridge/bridge_store.go @@ -0,0 +1,210 @@ +package bridge + +import ( + "encoding/json" + "fmt" + "net" + + "github.com/Sirupsen/logrus" + "github.com/docker/libkv/store" + "github.com/docker/libkv/store/boltdb" + "github.com/docker/libnetwork/datastore" + "github.com/docker/libnetwork/netlabel" + "github.com/docker/libnetwork/types" +) + +const bridgePrefix = "bridge" + +func (d *driver) initStore(option map[string]interface{}) error { + var err error + + provider, provOk := option[netlabel.LocalKVProvider] + provURL, urlOk := option[netlabel.LocalKVProviderURL] + + if provOk && urlOk { + cfg := &datastore.ScopeCfg{ + Client: datastore.ScopeClientCfg{ + Provider: provider.(string), + Address: provURL.(string), + }, + } + + provConfig, confOk := option[netlabel.LocalKVProviderConfig] + if confOk { + cfg.Client.Config = provConfig.(*store.Config) + } + + d.store, err = datastore.NewDataStore(datastore.LocalScope, cfg) + if err != nil { + return fmt.Errorf("bridge driver failed to initialize data store: %v", err) + } + + return d.populateNetworks() + } + + return nil +} + +func (d *driver) populateNetworks() error { + kvol, err := d.store.List(datastore.Key(bridgePrefix), &networkConfiguration{}) + if err != nil && err != datastore.ErrKeyNotFound && err != boltdb.ErrBoltBucketNotFound { + return fmt.Errorf("failed to get bridge network configurations from store: %v", err) + } + + // It's normal for network configuration state to be empty. Just return. + if err == datastore.ErrKeyNotFound { + return nil + } + + for _, kvo := range kvol { + ncfg := kvo.(*networkConfiguration) + if err = d.createNetwork(ncfg); err != nil { + logrus.Warnf("could not create bridge network for id %s bridge name %s while booting up from persistent state", ncfg.ID, ncfg.BridgeName) + } + } + + return nil +} + +func (d *driver) storeUpdate(kvObject datastore.KVObject) error { + if d.store == nil { + logrus.Warnf("bridge store not initialized. kv object %s is not added to the store", datastore.Key(kvObject.Key()...)) + return nil + } + + if err := d.store.PutObjectAtomic(kvObject); err != nil { + return fmt.Errorf("failed to update bridge store for object type %T: %v", kvObject, err) + } + + return nil +} + +func (d *driver) storeDelete(kvObject datastore.KVObject) error { + if d.store == nil { + logrus.Debugf("bridge store not initialized. kv object %s is not deleted from store", datastore.Key(kvObject.Key()...)) + return nil + } + +retry: + if err := d.store.DeleteObjectAtomic(kvObject); err != nil { + if err == datastore.ErrKeyModified { + if err := d.store.GetObject(datastore.Key(kvObject.Key()...), kvObject); err != nil { + return fmt.Errorf("could not update the kvobject to latest when trying to delete: %v", err) + } + goto retry + } + return err + } + + return nil +} + +func (ncfg *networkConfiguration) MarshalJSON() ([]byte, error) { + nMap := make(map[string]interface{}) + nMap["ID"] = ncfg.ID + nMap["BridgeName"] = ncfg.BridgeName + nMap["EnableIPv6"] = ncfg.EnableIPv6 + nMap["EnableIPMasquerade"] = ncfg.EnableIPMasquerade + nMap["EnableICC"] = ncfg.EnableICC + nMap["Mtu"] = ncfg.Mtu + nMap["DefaultBridge"] = ncfg.DefaultBridge + nMap["DefaultBindingIP"] = ncfg.DefaultBindingIP.String() + nMap["DefaultGatewayIPv4"] = ncfg.DefaultGatewayIPv4.String() + nMap["DefaultGatewayIPv6"] = ncfg.DefaultGatewayIPv6.String() + + if ncfg.AddressIPv4 != nil { + nMap["AddressIPv4"] = ncfg.AddressIPv4.String() + } + + if ncfg.AddressIPv6 != nil { + nMap["AddressIPv6"] = ncfg.AddressIPv6.String() + } + + return json.Marshal(nMap) +} + +func (ncfg *networkConfiguration) UnmarshalJSON(b []byte) error { + var ( + err error + nMap map[string]interface{} + ) + + if err = json.Unmarshal(b, &nMap); err != nil { + return err + } + + if v, ok := nMap["AddressIPv4"]; ok { + if ncfg.AddressIPv4, err = types.ParseCIDR(v.(string)); err != nil { + return types.InternalErrorf("failed to decode bridge network address IPv4 after json unmarshal: %s", v.(string)) + } + } + + if v, ok := nMap["AddressIPv6"]; ok { + if ncfg.AddressIPv6, err = types.ParseCIDR(v.(string)); err != nil { + return types.InternalErrorf("failed to decode bridge network address IPv6 after json unmarshal: %s", v.(string)) + } + } + + ncfg.DefaultBindingIP = net.ParseIP(nMap["DefaultBindingIP"].(string)) + ncfg.DefaultGatewayIPv4 = net.ParseIP(nMap["DefaultGatewayIPv4"].(string)) + ncfg.DefaultGatewayIPv6 = net.ParseIP(nMap["DefaultGatewayIPv6"].(string)) + ncfg.ID = nMap["ID"].(string) + ncfg.BridgeName = nMap["BridgeName"].(string) + ncfg.EnableIPv6 = nMap["EnableIPv6"].(bool) + ncfg.EnableIPMasquerade = nMap["EnableIPMasquerade"].(bool) + ncfg.EnableICC = nMap["EnableICC"].(bool) + ncfg.Mtu = int(nMap["Mtu"].(float64)) + + return nil +} + +func (ncfg *networkConfiguration) Key() []string { + return []string{bridgePrefix, ncfg.ID} +} + +func (ncfg *networkConfiguration) KeyPrefix() []string { + return []string{bridgePrefix} +} + +func (ncfg *networkConfiguration) Value() []byte { + b, err := json.Marshal(ncfg) + if err != nil { + return nil + } + return b +} + +func (ncfg *networkConfiguration) SetValue(value []byte) error { + return json.Unmarshal(value, ncfg) +} + +func (ncfg *networkConfiguration) Index() uint64 { + return ncfg.dbIndex +} + +func (ncfg *networkConfiguration) SetIndex(index uint64) { + ncfg.dbIndex = index + ncfg.dbExists = true +} + +func (ncfg *networkConfiguration) Exists() bool { + return ncfg.dbExists +} + +func (ncfg *networkConfiguration) Skip() bool { + return false +} + +func (ncfg *networkConfiguration) New() datastore.KVObject { + return &networkConfiguration{} +} + +func (ncfg *networkConfiguration) CopyTo(o datastore.KVObject) error { + dstNcfg := o.(*networkConfiguration) + *dstNcfg = *ncfg + return nil +} + +func (ncfg *networkConfiguration) DataScope() string { + return datastore.LocalScope +} diff --git a/drivers/bridge/bridge_test.go b/drivers/bridge/bridge_test.go index 40ce65d..e1891af 100644 --- a/drivers/bridge/bridge_test.go +++ b/drivers/bridge/bridge_test.go @@ -120,14 +120,6 @@ func TestCreate(t *testing.T) { if _, ok := err.(types.ForbiddenError); !ok { t.Fatalf("Creation of second network with default name failed with unexpected error type") } - - err = d.DeleteNetwork("dummy") - if err == nil { - t.Fatalf("deletion of network with default name should fail on this driver") - } - if _, ok := err.(types.ForbiddenError); !ok { - t.Fatalf("deletion of network with default name failed with unexpected error type") - } } func TestCreateFail(t *testing.T) { diff --git a/ipam/allocator_test.go b/ipam/allocator_test.go index df3b01f..e38d08c 100644 --- a/ipam/allocator_test.go +++ b/ipam/allocator_test.go @@ -32,7 +32,6 @@ func randomLocalStore() (datastore.DataStore, error) { return nil, fmt.Errorf("Error closing temp file: %v", err) } return datastore.NewDataStore(datastore.LocalScope, &datastore.ScopeCfg{ - Embedded: true, Client: datastore.ScopeClientCfg{ Provider: "boltdb", Address: defaultPrefix + tmp.Name(), diff --git a/network.go b/network.go index c58dd82..4537273 100644 --- a/network.go +++ b/network.go @@ -557,17 +557,6 @@ func (n *network) deleteNetwork() error { return fmt.Errorf("failed deleting network: %v", err) } - // If it is bridge network type make sure we call the driver about the network - // because the network may have been created in some past life of libnetwork. - if n.Type() == "bridge" { - n.drvOnce.Do(func() { - err = n.getController().addNetwork(n) - }) - if err != nil { - return err - } - } - if err := d.DeleteNetwork(n.ID()); err != nil { // Forbidden Errors should be honored if _, ok := err.(types.ForbiddenError); ok { @@ -585,17 +574,6 @@ func (n *network) addEndpoint(ep *endpoint) error { return fmt.Errorf("failed to add endpoint: %v", err) } - // If it is bridge network type make sure we call the driver about the network - // because the network may have been created in some past life of libnetwork. - if n.Type() == "bridge" { - n.drvOnce.Do(func() { - err = n.getController().addNetwork(n) - }) - if err != nil { - return err - } - } - err = d.CreateEndpoint(n.id, ep.id, ep.Interface(), ep.generic) if err != nil { return types.InternalErrorf("failed to create endpoint %s on network %s: %v",