Fix rebooting cluster from complete outage (#94)

Implement recovery from complete outage in SQL.

Currently dba.reboot_cluster_from_complete_outage() does not respect
the SSL mode of the cluster (see: https://bugs.mysql.com/90793). This
works around that until 8.0.12 is released.

Author: prydie

hack/mysqlsh-py.sh renamed to hack/mysqlsh.sh

@@ -2,8 +2,9 @@
 #
 # Gets an interactive Python based mysqlsh on the given instance.
 
-if [ "$#" -ne 1 ]; then
-    echo "Usage: $0 <namespace/podname>"
+if [ "$#" -lt 1 ]; then
+    echo "Usage: $0 <namespace/podname> [args]"
+    echo "example: $0 default/my-cluster-0 --py"
     exit 1
 fi
 
@@ -17,4 +18,4 @@ kubectl exec \
     -it \
     -c mysql-agent \
     ${POD} -- /bin/sh \
-    -c "PS1='\u@\h:\w\$ ' mysqlsh --no-wizard --uri ${URI} --py"
+    -c "PS1='\u@\h:\w\$ ' mysqlsh --no-wizard --uri ${URI} ${@:2}"

pkg/util/mysqlsh/mysqlsh.go

@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"regexp"
+	"strings"
 	"sync"
 
 	"github.com/golang/glog"
@@ -171,8 +172,36 @@ func (r *runner) run(ctx context.Context, python string) ([]byte, error) {
 }
 
 func (r *runner) RebootClusterFromCompleteOutage(ctx context.Context) error {
-	python := fmt.Sprintf("dba.reboot_cluster_from_complete_outage('%s')", innodb.DefaultClusterName)
-	_, err := r.run(ctx, python)
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	stdout := &bytes.Buffer{}
+	stderr := &bytes.Buffer{}
+
+	// NOTE(apryde): This is implemented in SQL rather than as a call to
+	// dba.reboot_cluster_from_complete_outage() due to https://bugs.mysql.com/90793.
+	sql := strings.Join([]string{
+		"RESET PERSIST group_replication_bootstrap_group;",
+		"SET GLOBAL group_replication_bootstrap_group=ON;",
+		"start group_replication;",
+	}, " ")
+
+	args := []string{"--no-wizard", "--uri", r.uri, "--sql", "-e", sql}
+
+	cmd := r.exec.CommandContext(ctx, "mysqlsh", args...)
+
+	cmd.SetStdout(stdout)
+	cmd.SetStderr(stderr)
+
+	glog.V(6).Infof("Running command: mysqlsh %v", args)
+	err := cmd.Run()
+	glog.V(6).Infof("    stdout: %s\n    stderr: %s\n    err: %s", stdout, stderr, err)
+	if err != nil {
+		underlying := NewErrorFromStderr(stderr.String())
+		if underlying != nil {
+			return errors.WithStack(underlying)
+		}
+	}
 	return err
 }