From fb2e843e99e28088a217bbadd280e7574bd30bac Mon Sep 17 00:00:00 2001 From: Nathan LeClaire Date: Fri, 17 Jul 2015 18:02:02 -0700 Subject: [PATCH] - Re-introduce config migration; fix panics occurring from older configs - Introduce boilerplate for config.json migrations Signed-off-by: Nathan LeClaire --- drivers/openstack/client.go | 2 +- drivers/rackspace/client.go | 2 +- libmachine/filestore.go | 16 +- libmachine/host.go | 84 +++++----- libmachine/host_v0.go | 34 ++++ libmachine/migrate.go | 157 ++++++------------ libmachine/migrate_v0_v1.go | 113 +++++++++++++ ...{migrate_test.go => migrate_v0_v1_test.go} | 22 +-- main.go | 2 +- script/build | 2 +- version/version.go | 13 +- 11 files changed, 270 insertions(+), 177 deletions(-) create mode 100644 libmachine/host_v0.go create mode 100644 libmachine/migrate_v0_v1.go rename libmachine/{migrate_test.go => migrate_v0_v1_test.go} (85%) diff --git a/drivers/openstack/client.go b/drivers/openstack/client.go index c2964512e4..72caa262df 100644 --- a/drivers/openstack/client.go +++ b/drivers/openstack/client.go @@ -437,7 +437,7 @@ func (c *GenericClient) Authenticate(d *Driver) error { return err } - provider.UserAgent.Prepend(fmt.Sprintf("docker-machine/v%s", version.VERSION)) + provider.UserAgent.Prepend(fmt.Sprintf("docker-machine/v%s", version.Version)) if d.Insecure { // Configure custom TLS settings. diff --git a/drivers/rackspace/client.go b/drivers/rackspace/client.go index bca5a784cf..c506a4578c 100644 --- a/drivers/rackspace/client.go +++ b/drivers/rackspace/client.go @@ -42,7 +42,7 @@ func (c *Client) Authenticate(d *openstack.Driver) error { return err } - provider.UserAgent.Prepend(fmt.Sprintf("docker-machine/v%s", version.VERSION)) + provider.UserAgent.Prepend(fmt.Sprintf("docker-machine/v%s", version.Version)) err = rackspace.Authenticate(provider, opts) if err != nil { diff --git a/libmachine/filestore.go b/libmachine/filestore.go index 7cdebb6394..d2c4115848 100644 --- a/libmachine/filestore.go +++ b/libmachine/filestore.go @@ -19,7 +19,11 @@ type Filestore struct { } func NewFilestore(rootPath string, caCert string, privateKey string) *Filestore { - return &Filestore{path: rootPath, caCertPath: caCert, privateKeyPath: privateKey} + return &Filestore{ + path: rootPath, + caCertPath: caCert, + privateKeyPath: privateKey, + } } func (s Filestore) loadHost(name string) (*Host, error) { @@ -30,13 +34,15 @@ func (s Filestore) loadHost(name string) (*Host, error) { } } - host := &Host{Name: name, StorePath: hostPath} + host := &Host{ + Name: name, + StorePath: hostPath, + } if err := host.LoadConfig(); err != nil { return nil, err } - h := FillNestedHost(host) - return h, nil + return host, nil } func (s Filestore) GetPath() string { @@ -85,7 +91,7 @@ func (s Filestore) List() ([]*Host, error) { for _, file := range dir { // don't load hidden dirs; used for configs - if file.IsDir() && strings.Index(file.Name(), ".") != 0 { + if file.IsDir() && !strings.HasPrefix(file.Name(), ".") { host, err := s.Get(file.Name()) if err != nil { log.Errorf("error loading host %q: %s", file.Name(), err) diff --git a/libmachine/host.go b/libmachine/host.go index 83a1b46a61..671f150e57 100644 --- a/libmachine/host.go +++ b/libmachine/host.go @@ -19,6 +19,7 @@ import ( "github.com/docker/machine/ssh" "github.com/docker/machine/state" "github.com/docker/machine/utils" + "github.com/docker/machine/version" ) var ( @@ -28,22 +29,12 @@ var ( ) type Host struct { - Name string `json:"-"` - DriverName string - Driver drivers.Driver - StorePath string - HostOptions *HostOptions - - // deprecated options; these are left to assist in config migrations - SwarmHost string - SwarmMaster bool - SwarmDiscovery string - CaCertPath string - PrivateKeyPath string - ServerCertPath string - ServerKeyPath string - ClientCertPath string - ClientKeyPath string + ConfigVersion int + Driver drivers.Driver + DriverName string + HostOptions *HostOptions + Name string `json:"-"` + StorePath string } type HostOptions struct { @@ -56,14 +47,9 @@ type HostOptions struct { } type HostMetadata struct { - DriverName string - HostOptions HostOptions - StorePath string - CaCertPath string - PrivateKeyPath string - ServerCertPath string - ServerKeyPath string - ClientCertPath string + ConfigVersion int + DriverName string + HostOptions HostOptions } type HostListItem struct { @@ -75,6 +61,14 @@ type HostListItem struct { SwarmOptions swarm.SwarmOptions } +type ErrSavingConfig struct { + wrappedErr error +} + +func (e ErrSavingConfig) Error() string { + return fmt.Sprintf("Error saving config: %s", e.wrappedErr) +} + func NewHost(name, driverName string, hostOptions *HostOptions) (*Host, error) { authOptions := hostOptions.AuthOptions storePath := filepath.Join(utils.GetMachineDir(), name) @@ -83,11 +77,12 @@ func NewHost(name, driverName string, hostOptions *HostOptions) (*Host, error) { return nil, err } return &Host{ - Name: name, - DriverName: driverName, - Driver: driver, - StorePath: storePath, - HostOptions: hostOptions, + Name: name, + ConfigVersion: version.ConfigVersion, + DriverName: driverName, + Driver: driver, + StorePath: storePath, + HostOptions: hostOptions, }, nil } @@ -100,6 +95,7 @@ func LoadHost(name string, StorePath string) (*Host, error) { if err := host.LoadConfig(); err != nil { return nil, err } + return host, nil } @@ -287,26 +283,28 @@ func (h *Host) LoadConfig() error { return err } - // First pass: find the driver name and load the driver - var hostMetadata HostMetadata - if err := json.Unmarshal(data, &hostMetadata); err != nil { - return err - } + // Remember the machine name and store path so we don't have to pass it + // through each struct in the migration. + name := h.Name + storePath := h.StorePath - meta := FillNestedHostMetadata(&hostMetadata) + // If we end up performing a migration, we should save afterwards so we don't have to do it again on subsequent invocations. + migrationPerformed := h.ConfigVersion != version.ConfigVersion - authOptions := meta.HostOptions.AuthOptions - - driver, err := drivers.NewDriver(hostMetadata.DriverName, h.Name, h.StorePath, authOptions.CaCertPath, authOptions.PrivateKeyPath) + migratedHost, err := MigrateHost(h, data) if err != nil { - return err + return fmt.Errorf("Error getting migrated host: %s", err) } - h.Driver = driver + *h = *migratedHost - // Second pass: unmarshal driver config into correct driver - if err := json.Unmarshal(data, &h); err != nil { - return err + h.Name = name + h.StorePath = storePath + + if migrationPerformed { + if err := h.SaveConfig(); err != nil { + return fmt.Errorf("Error saving config after migration was performed: %s", err) + } } return nil diff --git a/libmachine/host_v0.go b/libmachine/host_v0.go new file mode 100644 index 0000000000..8c1d63839d --- /dev/null +++ b/libmachine/host_v0.go @@ -0,0 +1,34 @@ +package libmachine + +import "github.com/docker/machine/drivers" + +type HostV0 struct { + Name string `json:"-"` + Driver drivers.Driver + DriverName string + ConfigVersion int + HostOptions *HostOptions + + StorePath string + CaCertPath string + PrivateKeyPath string + ServerCertPath string + ServerKeyPath string + ClientCertPath string + SwarmHost string + SwarmMaster bool + SwarmDiscovery string + ClientKeyPath string +} + +type HostMetadataV0 struct { + HostOptions HostOptions + DriverName string + + StorePath string + CaCertPath string + PrivateKeyPath string + ServerCertPath string + ServerKeyPath string + ClientCertPath string +} diff --git a/libmachine/migrate.go b/libmachine/migrate.go index 69f062e016..80555ec810 100644 --- a/libmachine/migrate.go +++ b/libmachine/migrate.go @@ -1,122 +1,65 @@ package libmachine import ( - "path/filepath" + "encoding/json" + "fmt" - "github.com/docker/machine/libmachine/auth" - "github.com/docker/machine/libmachine/engine" - "github.com/docker/machine/libmachine/swarm" - "github.com/docker/machine/utils" + "github.com/docker/machine/drivers" + "github.com/docker/machine/version" ) -// In the 0.0.1 => 0.0.2 transition, the JSON representation of -// machines changed from a "flat" to a more "nested" structure -// for various options and configuration settings. To preserve -// compatibility with existing machines, these migration functions -// have been introduced. They preserve backwards compat at the expense -// of some duplicated information. +func getMigratedHostMetadata(data []byte) (*HostMetadata, error) { + // HostMetadata is for a "first pass" so we can then load the driver + var ( + hostMetadata *HostMetadataV0 + ) -// validates host config and modifies if needed -// this is used for configuration updates -func FillNestedHost(host *Host) *Host { - certInfo := getCertInfoFromHost(host) - - if host.HostOptions == nil { - host.HostOptions = &HostOptions{} - } - if host.HostOptions.EngineOptions == nil { - host.HostOptions.EngineOptions = &engine.EngineOptions{} + if err := json.Unmarshal(data, &hostMetadata); err != nil { + return &HostMetadata{}, err } - if host.HostOptions.SwarmOptions == nil { - host.HostOptions.SwarmOptions = &swarm.SwarmOptions{ - Address: "", - Discovery: host.SwarmDiscovery, - Host: host.SwarmHost, - Master: host.SwarmMaster, + migratedHostMetadata := MigrateHostMetadataV0ToHostMetadataV1(hostMetadata) + + return migratedHostMetadata, nil +} + +func MigrateHost(h *Host, data []byte) (*Host, error) { + migratedHostMetadata, err := getMigratedHostMetadata(data) + if err != nil { + return &Host{}, err + } + + authOptions := migratedHostMetadata.HostOptions.AuthOptions + + driver, err := drivers.NewDriver( + migratedHostMetadata.DriverName, + h.Name, + h.StorePath, + authOptions.CaCertPath, + authOptions.PrivateKeyPath, + ) + if err != nil { + return &Host{}, err + } + + for h.ConfigVersion = migratedHostMetadata.ConfigVersion; h.ConfigVersion < version.ConfigVersion; h.ConfigVersion++ { + switch h.ConfigVersion { + case 0: + hostV0 := &HostV0{ + Driver: driver, + } + if err := json.Unmarshal(data, &hostV0); err != nil { + return &Host{}, fmt.Errorf("Error unmarshalling host config version 0: %s", err) + } + h = MigrateHostV0ToHostV1(hostV0) + default: } } - host.HostOptions.AuthOptions = &auth.AuthOptions{ - StorePath: host.StorePath, - CaCertPath: certInfo.CaCertPath, - CaCertRemotePath: "", - ServerCertPath: certInfo.ServerCertPath, - ServerKeyPath: certInfo.ServerKeyPath, - ClientKeyPath: certInfo.ClientKeyPath, - ServerCertRemotePath: "", - ServerKeyRemotePath: "", - PrivateKeyPath: certInfo.CaKeyPath, - ClientCertPath: certInfo.ClientCertPath, + h.Driver = driver + if err := json.Unmarshal(data, &h); err != nil { + return &Host{}, fmt.Errorf("Error unmarshalling most recent host version: %s", err) } - return host -} - -// fills nested host metadata and modifies if needed -// this is used for configuration updates -func FillNestedHostMetadata(m *HostMetadata) *HostMetadata { - if m.HostOptions.EngineOptions == nil { - m.HostOptions.EngineOptions = &engine.EngineOptions{} - } - - if m.HostOptions.AuthOptions == nil { - m.HostOptions.AuthOptions = &auth.AuthOptions{ - StorePath: m.StorePath, - CaCertPath: m.CaCertPath, - CaCertRemotePath: "", - ServerCertPath: m.ServerCertPath, - ServerKeyPath: m.ServerKeyPath, - ClientKeyPath: "", - ServerCertRemotePath: "", - ServerKeyRemotePath: "", - PrivateKeyPath: m.PrivateKeyPath, - ClientCertPath: m.ClientCertPath, - } - } - - return m -} - -func getCertInfoFromHost(h *Host) CertPathInfo { - // setup cert paths - caCertPath := h.CaCertPath - caKeyPath := h.PrivateKeyPath - clientCertPath := h.ClientCertPath - clientKeyPath := h.ClientKeyPath - serverCertPath := h.ServerCertPath - serverKeyPath := h.ServerKeyPath - - if caCertPath == "" { - caCertPath = filepath.Join(utils.GetMachineCertDir(), "ca.pem") - } - - if caKeyPath == "" { - caKeyPath = filepath.Join(utils.GetMachineCertDir(), "ca-key.pem") - } - - if clientCertPath == "" { - clientCertPath = filepath.Join(utils.GetMachineCertDir(), "cert.pem") - } - - if clientKeyPath == "" { - clientKeyPath = filepath.Join(utils.GetMachineCertDir(), "key.pem") - } - - if serverCertPath == "" { - serverCertPath = filepath.Join(utils.GetMachineCertDir(), "server.pem") - } - - if serverKeyPath == "" { - serverKeyPath = filepath.Join(utils.GetMachineCertDir(), "server-key.pem") - } - - return CertPathInfo{ - CaCertPath: caCertPath, - CaKeyPath: caKeyPath, - ClientCertPath: clientCertPath, - ClientKeyPath: clientKeyPath, - ServerCertPath: serverCertPath, - ServerKeyPath: serverKeyPath, - } + return h, nil } diff --git a/libmachine/migrate_v0_v1.go b/libmachine/migrate_v0_v1.go new file mode 100644 index 0000000000..61916da407 --- /dev/null +++ b/libmachine/migrate_v0_v1.go @@ -0,0 +1,113 @@ +package libmachine + +import ( + "path/filepath" + + "github.com/docker/machine/libmachine/auth" + "github.com/docker/machine/libmachine/engine" + "github.com/docker/machine/libmachine/swarm" + "github.com/docker/machine/utils" +) + +// In the 0.0.1 => 0.0.2 transition, the JSON representation of +// machines changed from a "flat" to a more "nested" structure +// for various options and configuration settings. To preserve +// compatibility with existing machines, these migration functions +// have been introduced. They preserve backwards compat at the expense +// of some duplicated information. + +// validates host config and modifies if needed +// this is used for configuration updates +func MigrateHostV0ToHostV1(hostV0 *HostV0) *Host { + host := &Host{} + + certInfoV0 := getCertInfoFromHost(hostV0) + + host.HostOptions = &HostOptions{} + host.HostOptions.EngineOptions = &engine.EngineOptions{} + host.HostOptions.SwarmOptions = &swarm.SwarmOptions{ + Address: "", + Discovery: hostV0.SwarmDiscovery, + Host: hostV0.SwarmHost, + Master: hostV0.SwarmMaster, + } + host.HostOptions.AuthOptions = &auth.AuthOptions{ + StorePath: hostV0.StorePath, + CaCertPath: certInfoV0.CaCertPath, + CaCertRemotePath: "", + ServerCertPath: certInfoV0.ServerCertPath, + ServerKeyPath: certInfoV0.ServerKeyPath, + ClientKeyPath: certInfoV0.ClientKeyPath, + ServerCertRemotePath: "", + ServerKeyRemotePath: "", + PrivateKeyPath: certInfoV0.CaKeyPath, + ClientCertPath: certInfoV0.ClientCertPath, + } + + return host +} + +// fills nested host metadata and modifies if needed +// this is used for configuration updates +func MigrateHostMetadataV0ToHostMetadataV1(m *HostMetadataV0) *HostMetadata { + hostMetadata := &HostMetadata{} + hostMetadata.DriverName = m.DriverName + hostMetadata.HostOptions.EngineOptions = &engine.EngineOptions{} + hostMetadata.HostOptions.AuthOptions = &auth.AuthOptions{ + StorePath: m.StorePath, + CaCertPath: m.CaCertPath, + CaCertRemotePath: "", + ServerCertPath: m.ServerCertPath, + ServerKeyPath: m.ServerKeyPath, + ClientKeyPath: "", + ServerCertRemotePath: "", + ServerKeyRemotePath: "", + PrivateKeyPath: m.PrivateKeyPath, + ClientCertPath: m.ClientCertPath, + } + + return hostMetadata +} + +func getCertInfoFromHost(h *HostV0) CertPathInfo { + // setup cert paths + caCertPath := h.CaCertPath + caKeyPath := h.PrivateKeyPath + clientCertPath := h.ClientCertPath + clientKeyPath := h.ClientKeyPath + serverCertPath := h.ServerCertPath + serverKeyPath := h.ServerKeyPath + + if caCertPath == "" { + caCertPath = filepath.Join(utils.GetMachineCertDir(), "ca.pem") + } + + if caKeyPath == "" { + caKeyPath = filepath.Join(utils.GetMachineCertDir(), "ca-key.pem") + } + + if clientCertPath == "" { + clientCertPath = filepath.Join(utils.GetMachineCertDir(), "cert.pem") + } + + if clientKeyPath == "" { + clientKeyPath = filepath.Join(utils.GetMachineCertDir(), "key.pem") + } + + if serverCertPath == "" { + serverCertPath = filepath.Join(utils.GetMachineCertDir(), "server.pem") + } + + if serverKeyPath == "" { + serverKeyPath = filepath.Join(utils.GetMachineCertDir(), "server-key.pem") + } + + return CertPathInfo{ + CaCertPath: caCertPath, + CaKeyPath: caKeyPath, + ClientCertPath: clientCertPath, + ClientKeyPath: clientKeyPath, + ServerCertPath: serverCertPath, + ServerKeyPath: serverKeyPath, + } +} diff --git a/libmachine/migrate_test.go b/libmachine/migrate_v0_v1_test.go similarity index 85% rename from libmachine/migrate_test.go rename to libmachine/migrate_v0_v1_test.go index 57c726cf1f..9b6cacb15b 100644 --- a/libmachine/migrate_test.go +++ b/libmachine/migrate_v0_v1_test.go @@ -10,9 +10,9 @@ import ( "github.com/docker/machine/libmachine/swarm" ) -func TestFillNestedHost(t *testing.T) { +func TestMigrateHostV0ToV1(t *testing.T) { os.Setenv("MACHINE_STORAGE_PATH", "/tmp/migration") - originalHost := &Host{ + originalHost := &HostV0{ HostOptions: nil, SwarmDiscovery: "token://foobar", SwarmHost: "1.2.3.4:2376", @@ -42,13 +42,10 @@ func TestFillNestedHost(t *testing.T) { } expectedHost := &Host{ - SwarmHost: "1.2.3.4:2376", - SwarmDiscovery: "token://foobar", - SwarmMaster: true, - HostOptions: hostOptions, + HostOptions: hostOptions, } - host := FillNestedHost(originalHost) + host := MigrateHostV0ToHostV1(originalHost) if !reflect.DeepEqual(host, expectedHost) { t.Logf("\n%+v\n%+v", host, expectedHost) @@ -57,8 +54,8 @@ func TestFillNestedHost(t *testing.T) { } } -func TestFillNestedHostMetadata(t *testing.T) { - metadata := &HostMetadata{ +func TestMigrateHostMetadataV0ToV1(t *testing.T) { + metadata := &HostMetadataV0{ HostOptions: HostOptions{ EngineOptions: nil, AuthOptions: nil, @@ -78,12 +75,9 @@ func TestFillNestedHostMetadata(t *testing.T) { EngineOptions: &engine.EngineOptions{}, AuthOptions: expectedAuthOptions, }, - StorePath: "/tmp/store", - CaCertPath: "/tmp/store/certs/ca.pem", - ServerCertPath: "/tmp/store/certs/server.pem", } - m := FillNestedHostMetadata(metadata) + m := MigrateHostMetadataV0ToHostMetadataV1(metadata) if !reflect.DeepEqual(m, expectedMetadata) { t.Logf("\n%+v\n%+v", m, expectedMetadata) @@ -95,7 +89,7 @@ func TestFillNestedHostMetadata(t *testing.T) { // due to a schema migration from "flat" to a "nested" structure. func TestGetCertInfoFromHost(t *testing.T) { os.Setenv("MACHINE_STORAGE_PATH", "/tmp/migration") - host := &Host{ + host := &HostV0{ CaCertPath: "", PrivateKeyPath: "", ClientCertPath: "", diff --git a/main.go b/main.go index a37bf1d908..7f640babd0 100644 --- a/main.go +++ b/main.go @@ -67,7 +67,7 @@ func main() { app.Commands = commands.Commands app.CommandNotFound = cmdNotFound app.Usage = "Create and manage machines running Docker." - app.Version = version.VERSION + " (" + version.GITCOMMIT + ")" + app.Version = version.Version + " (" + version.GitCommit + ")" app.Flags = []cli.Flag{ cli.BoolFlag{ diff --git a/script/build b/script/build index 07efc57066..385333db23 100755 --- a/script/build +++ b/script/build @@ -21,4 +21,4 @@ fi # Get rid of existing binaries rm -f docker-machine* rm -rf Godeps/_workspace/pkg -docker run --rm -v `pwd`:/go/src/github.com/docker/machine docker-machine gox "${OS_PLATFORM_ARG[@]}" "${OS_ARCH_ARG[@]}" -output="docker-machine_{{.OS}}-{{.Arch}}" -ldflags="-w -X github.com/docker/machine/version.GITCOMMIT `git rev-parse --short HEAD`" +docker run --rm -v `pwd`:/go/src/github.com/docker/machine docker-machine gox "${OS_PLATFORM_ARG[@]}" "${OS_ARCH_ARG[@]}" -output="docker-machine_{{.OS}}-{{.Arch}}" -ldflags="-w -X github.com/docker/machine/version.GitCommit `git rev-parse --short HEAD`" diff --git a/version/version.go b/version/version.go index e1e0d6b9a9..1fef2e53db 100644 --- a/version/version.go +++ b/version/version.go @@ -1,9 +1,14 @@ package version var ( - // VERSION should be updated by hand at each release - VERSION = "0.4.0-dev" + // ConfigVersion dictates which version of the config.json format is + // used. It needs to be bumped if there is a breaking change, and + // therefore migration, introduced to the config file format. + ConfigVersion = 1 - // GITCOMMIT will be overwritten automatically by the build system - GITCOMMIT = "HEAD" + // Version should be updated by hand at each release + Version = "0.4.0-dev" + + // GitCommit will be overwritten automatically by the build system + GitCommit = "HEAD" )