Upgrade to MySQL 8.0.11 (GA) (#87)

* Migrate to MySQL 8.0.11
* Implement recovery from complete outage
* Remove 5.7.20-1.1.2 as a valid version
* Remove clearing of binary logs on add instance
* Use name-index.name to address cluster members (fixes #84)
* Stop leaking mysqlsh processes
* Disable GR SSL
* Whitelist ips for GR members

Author: prydie

Makefile

@@ -17,12 +17,12 @@ USE_GLOBAL_NAMESPACE ?= false
 ifdef WERCKER
     # Insert swear words about mysql group replication and hostname length. Arghh..
     NEW_NAMESPACE ?= e2e-$(shell echo ${WERCKER_GIT_COMMIT} | fold -w 8 | head -n1)
-    VERSION ?= ${WERCKER_GIT_COMMIT}
+    VERSION := ${WERCKER_GIT_COMMIT}
     E2E_FUNC := e2efunc-wercker
     E2E_NON_BUFFERED_LOGS ?= false
 else
     NEW_NAMESPACE ?= e2e-${USER}
-    VERSION ?= ${USER}-$(shell date +%Y%m%d%H%M%S)
+    VERSION := ${USER}-$(shell date +%Y%m%d%H%M%S)
     E2E_FUNC := e2efunc-docker
     E2E_NON_BUFFERED_LOGS ?= true
 endif

cmd/mysql-agent/app/mysql_agent.go

@@ -69,12 +69,12 @@ func Run(opts *options.MySQLAgentOpts) error {
 	signals.SetupSignalHandler(cancelFunc)
 
 	// Set up healthchecks (liveness and readiness).
+	checkInCluster, err := cluster.NewHealthCheck(ctx)
+	if err != nil {
+		glog.Fatal(err)
+	}
 	health := healthcheck.NewHandler()
-	health.AddReadinessCheck("node-in-cluster",
-		healthcheck.Async(
-			healthcheck.Timeout(func() error { return cluster.CheckNodeInCluster(ctx) }, 5*time.Second),
-			10*time.Second,
-		))
+	health.AddReadinessCheck("node-in-cluster", checkInCluster)
 	go func() {
 		glog.Fatal(http.ListenAndServe(
 			net.JoinHostPort(opts.Address, strconv.Itoa(int(opts.HealthcheckPort))),

docker/mysql-agent/Dockerfile

@@ -1,4 +1,4 @@
-FROM mysql/mysql-server:5.7.20-1.1.2
+FROM mysql/mysql-server:8.0.11
 
 COPY bin/linux_amd64/mysql-agent /
 

examples/backup/backup-schedule.yaml

@@ -10,12 +10,12 @@ spec:
       databases:
         - test
     storage:
-      provider: s3  
+      provider: s3
       secretRef:
         name: s3-credentials
       config:
         endpoint: ocitenancy.compat.objectstorage.ociregion.oraclecloud.com
         region: ociregion
         bucket: mybucket
     clusterRef:
-      name: mysql-cluster
+      name: mysql

examples/backup/backup.yaml

@@ -8,12 +8,12 @@ spec:
     databases:
       - test
   storage:
-    provider: s3  
+    provider: s3
     secretRef:
       name: s3-credentials
-    config:					
+    config:
       endpoint: ocitenancy.compat.objectstorage.ociregion.oraclecloud.com
       region:   ociregion
       bucket:   mybucket
   clusterRef:
-    name: mysql-cluster
+    name: mysql

examples/cluster/cluster-with-3-replicas.yaml

@@ -1,6 +1,6 @@
 apiVersion: "mysql.oracle.com/v1"
 kind: MySQLCluster
 metadata:
-  name: mysql-cluster-with-3-replicas
+  name: mysql
 spec:
   replicas: 3

examples/cluster/cluster-with-custom-config.yaml

@@ -1,7 +1,7 @@
 apiVersion: "mysql.oracle.com/v1"
 kind: MySQLCluster
 metadata:
-  name: mysql-cluster-with-custom-config
+  name: mysql
 spec:
   replicas: 1
   configRef:

examples/cluster/cluster-with-custom-secret.yaml

@@ -1,7 +1,7 @@
 apiVersion: "mysql.oracle.com/v1"
 kind: MySQLCluster
 metadata:
-  name: mysql-cluster-with-custom-secret
+  name: mysql
 spec:
   replicas: 1
   secretRef:

examples/cluster/cluster-with-data-volume-and-backup-volume.yaml

@@ -34,7 +34,7 @@ spec:
 apiVersion: "mysql.oracle.com/v1"
 kind: MySQLCluster
 metadata:
-  name: mysql-cluster-with-data-volume-and-backup-volume
+  name: mysql
 spec:
   replicas: 1
   secretRef:

examples/cluster/cluster-with-volume.yaml

@@ -18,7 +18,7 @@ spec:
 apiVersion: "mysql.oracle.com/v1"
 kind: MySQLCluster
 metadata:
-  name: example-mysql-cluster-with-volume
+  name: mysql
 spec:
   replicas: 1
   secretRef:

examples/cluster/cluster.yaml

@@ -1,4 +1,4 @@
 apiVersion: "mysql.oracle.com/v1"
 kind: MySQLCluster
 metadata:
-  name: mysql-cluster
+  name: mysql

examples/cluster/multi-master-cluster.yaml

@@ -1,7 +1,7 @@
 apiVersion: "mysql.oracle.com/v1"
 kind: MySQLCluster
 metadata:
-  name: mysql-multimaster-cluster
+  name: mysql
 spec:
   multiMaster: true
   replicas: 3

examples/demo/wordpress-router/wordpress-database.yaml

@@ -1,9 +1,27 @@
 ---
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: mysql-config
+data:
+  my.cnf: |-
+    [mysqld]
+    default_authentication_plugin=mysql_native_password
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: wordpress-mysql-root-password
+data:
+  password: bXktc3VwZXItc2VjcmV0LXBhc3M=
+---
 apiVersion: "mysql.oracle.com/v1"
 kind: MySQLCluster
 metadata:
   name: mysql-wordpress
 spec:
   replicas: 3
+  configRef:
+    name: mysql-config
   secretRef:
     name: wordpress-mysql-root-password

examples/demo/wordpress-router/wordpress-deployment.yaml

@@ -28,7 +28,7 @@ spec:
     spec:
       containers:
       - name: mysqlrouter
-        image: mysql/mysql-router:8.0
+        image: mysql/mysql-router:8.0.11
         env:
         - name: MYSQL_PASSWORD
           valueFrom:
@@ -46,14 +46,9 @@ spec:
         command:
         - "/bin/bash"
         - "-cx"
-        - |
-          echo "Updating resolv.conf"
-          search=$(grep ^search /etc/resolv.conf)
-          echo "$search mysql-wordpress.default.svc.cluster.local" >> /etc/resolv.conf
-          echo "Running the router"
-          exec /run.sh mysqlrouter
+        - "exec /run.sh mysqlrouter"
       - name: wordpress
-        image: wordpress:4.8.0-apache
+        image: wordpress:4.9.0-apache
         env:
         - name: WORDPRESS_DB_HOST
           value: 127.0.0.1:6446

examples/restore/restore.yaml

@@ -4,6 +4,6 @@ metadata:
   name: mysql-restore
 spec:
   clusterRef:
-    name: mysql-cluster
+    name: mysql
   backupRef:
     name: mysql-backup

hack/mysql.sh

@@ -9,10 +9,12 @@ fi
 
 NAMESPACE=${1%/*}
 POD=${1#*/}
+CLUSTER_NAME=${POD%-*}  # statefulset and service name
+HOST="${POD}.${CLUSTER_NAME}"
 
 kubectl exec \
     -n ${NAMESPACE} \
     -it \
     -c mysql-agent \
     ${POD} -- /bin/sh \
-    -c 'mysql -uroot -p$MYSQL_ROOT_PASSWORD'
+    -c "mysql -uroot -p\$MYSQL_ROOT_PASSWORD -h ${HOST}"

hack/mysqlsh-py.sh

@@ -9,10 +9,12 @@ fi
 
 NAMESPACE=${1%/*}
 POD=${1#*/}
+CLUSTER_NAME=${POD%-*}  # statefulset and service name
+URI="root:\$MYSQL_ROOT_PASSWORD@${POD}.${CLUSTER_NAME}:3306"
 
 kubectl exec \
     -n ${NAMESPACE} \
     -it \
     -c mysql-agent \
     ${POD} -- /bin/sh \
-    -c 'mysqlsh --uri "root:$MYSQL_ROOT_PASSWORD@localhost:3306" --py'
+    -c "PS1='\u@\h:\w\$ ' mysqlsh --no-wizard --uri ${URI} --py"

pkg/apis/mysql/v1/cluster.go

@@ -22,7 +22,7 @@ import (
 
 const (
 	// The default MySQL version to use if not specified explicitly by user
-	defaultVersion      = "5.7.20-1.1.2"
+	defaultVersion      = "8.0.11"
 	defaultReplicas     = 1
 	defaultBaseServerID = 1000
 	// Max safe value for BaseServerID calculated as max MySQL server_id value - max Replication Group size
@@ -32,6 +32,17 @@ const (
 // ClusterCRDResourcePlural defines the custom resource name for mysqlclusters
 const ClusterCRDResourcePlural = "mysqlclusters"
 
+const (
+	// MaxInnoDBClusterMembers is the maximum number of members supported by InnoDB
+	// group replication.
+	MaxInnoDBClusterMembers = 9
+
+	// MySQLClusterNameMaxLen is the maximum supported length of a
+	// MySQLCluster name.
+	// See: https://bugs.mysql.com/bug.php?id=90601
+	MySQLClusterNameMaxLen = 28
+)
+
 // TODO (owain) we need to remove this because it's not reasonable for us to maintain a list
 // of all the potential MySQL versions that can be used and in reality, it shouldn't matter
 // too much. The burden of this is not worth the benfit to a user

pkg/apis/mysql/v1/validate_cluster.go

@@ -15,22 +15,45 @@
 package v1
 
 import (
+	"fmt"
 	"strconv"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 )
 
 func validateCluster(c *MySQLCluster) field.ErrorList {
 	allErrs := field.ErrorList{}
+	allErrs = append(allErrs, validateClusterMetadata(c.ObjectMeta, field.NewPath("metadata"))...)
 	allErrs = append(allErrs, validateClusterSpec(c.Spec, field.NewPath("spec"))...)
 	allErrs = append(allErrs, validateClusterStatus(c.Status, field.NewPath("status"))...)
 	return allErrs
 }
 
+func validateClusterMetadata(m metav1.ObjectMeta, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	allErrs = append(allErrs, validateName(m.Name, fldPath.Child("name"))...)
+
+	return allErrs
+}
+
+func validateName(name string, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+
+	if len(name) > MySQLClusterNameMaxLen {
+		msg := fmt.Sprintf("longer than maximum supported length %d (see: https://bugs.mysql.com/bug.php?id=90601)", MaxInnoDBClusterMembers)
+		allErrs = append(allErrs, field.Invalid(fldPath, name, msg))
+	}
+
+	return allErrs
+}
+
 func validateClusterSpec(s MySQLClusterSpec, fldPath *field.Path) field.ErrorList {
 	allErrs := field.ErrorList{}
 
 	allErrs = append(allErrs, validateVersion(s.Version, fldPath.Child("version"))...)
+	allErrs = append(allErrs, validateReplicas(s.Replicas, fldPath.Child("replicas"))...)
 	allErrs = append(allErrs, validateBaseServerID(s.BaseServerID, fldPath.Child("baseServerId"))...)
 
 	return allErrs
@@ -69,3 +92,11 @@ func validatePhase(phase MySQLClusterPhase, fldPath *field.Path) field.ErrorList
 	}
 	return append(allErrs, field.Invalid(fldPath, phase, "invalid phase specified"))
 }
+
+func validateReplicas(replicas int32, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if replicas < 1 || replicas > MaxInnoDBClusterMembers {
+		allErrs = append(allErrs, field.Invalid(fldPath, replicas, "InnoDB clustering supports between 1-9 members"))
+	}
+	return allErrs
+}

pkg/cluster/cluster_health.go

@@ -16,47 +16,57 @@ package cluster
 
 import (
 	"context"
-	"errors"
+	"sync"
+	"time"
 
-	"github.com/golang/glog"
-	utilexec "k8s.io/utils/exec"
+	"github.com/heptiolabs/healthcheck"
+	"github.com/pkg/errors"
 
 	"github.com/oracle/mysql-operator/pkg/cluster/innodb"
-	"github.com/oracle/mysql-operator/pkg/util/mysqlsh"
 )
 
-// CheckNodeInCluster checks whether or not the local MySQL instance is a member
-// of an InnoDB cluster.
-func CheckNodeInCluster(ctx context.Context) error {
-	instance, err := NewLocalInstance()
-	if err != nil {
-		return err
-	}
-	mysh := mysqlsh.New(utilexec.New(), instance.GetShellURI())
-	clusterStatus, err := mysh.GetClusterStatus(ctx)
-	if err != nil {
-		return err
-	}
-	if clusterStatus.GetInstanceStatus(instance.Name()) != innodb.InstanceStatusOnline {
-		return errors.New("database still requires management")
-	}
-	return nil
+var (
+	status      *innodb.ClusterStatus
+	statusMutex sync.Mutex
+)
+
+// SetStatus sets the status of the local mysql cluster. The cluster manager
+// controller is responsible for updating.
+func SetStatus(new *innodb.ClusterStatus) {
+	statusMutex.Lock()
+	defer statusMutex.Unlock()
+	status = new.DeepCopy()
 }
 
-// GetClusterStatus returns a JSON string representing the status of the InnoDb
-// MySQL cluster. TODO: Remove me.
-func GetClusterStatus(ctx context.Context) (*innodb.ClusterStatus, error) {
-	pod, err := NewLocalInstance()
-	if err != nil {
-		glog.Errorf("Failed to get the pod details: %+v", err)
-		return nil, err
+// GetStatus fetches a copy of the latest cluster status.
+func GetStatus() *innodb.ClusterStatus {
+	statusMutex.Lock()
+	defer statusMutex.Unlock()
+	if status == nil {
+		return nil
 	}
+	return status.DeepCopy()
+}
 
-	mysh := mysqlsh.New(utilexec.New(), pod.GetShellURI())
-	clusterStatus, err := mysh.GetClusterStatus(ctx)
+// NewHealthCheck constructs a healthcheck for the local instance which checks
+// cluster status using mysqlsh.
+func NewHealthCheck(ctx context.Context) (healthcheck.Check, error) {
+	instance, err := NewLocalInstance()
 	if err != nil {
-		glog.V(4).Info("Failed to get the cluster status")
-		return nil, err
+		return nil, errors.Wrap(err, "getting local mysql instance")
 	}
-	return clusterStatus, nil
+
+	return healthcheck.AsyncWithContext(ctx,
+		healthcheck.Timeout(
+			func() error {
+				s := GetStatus()
+				if s == nil || s.GetInstanceStatus(instance.Name()) != innodb.InstanceStatusOnline {
+					return errors.New("database still requires management")
+				}
+				return nil
+			},
+			5*time.Second,
+		),
+		10*time.Second,
+	), nil
 }