From: Tatsuo Ishii Date: Sun, 25 Aug 2019 01:54:33 +0000 (+0900) Subject: Doc: fix indentation in scripts. X-Git-Tag: V4_1_0_BETA1~23 X-Git-Url: http://git.postgresql.org/gitweb/?a=commitdiff_plain;h=7feebd59d56d50ad975d1b24f3102060a6e57513;p=pgpool2.git Doc: fix indentation in scripts. Auto indentation by commit 2cb0bd3f8f236aeacfba37cd4d604893561bad52 broke indentation of scripts in tag. --- diff --git a/doc.ja/src/sgml/example-cluster.sgml b/doc.ja/src/sgml/example-cluster.sgml index 40f9938c6..f731a3991 100644 --- a/doc.ja/src/sgml/example-cluster.sgml +++ b/doc.ja/src/sgml/example-cluster.sgml @@ -486,82 +486,82 @@ /etc/pgpool-II/failover.sh - #!/bin/bash - # This script is run by failover_command. - - set -o xtrace - exec > >(logger -i -p local1.info) 2>&1 - - # Special values: - # %d = failed node id - # %h = failed node hostname - # %p = failed node port number - # %D = failed node database cluster path - # %m = new master node id - # %H = new master node hostname - # %M = old master node id - # %P = old primary node id - # %r = new master port number - # %R = new master database cluster path - # %N = old primary node hostname - # %S = old primary node port number - # %% = '%' character - - FAILED_NODE_ID="$1" - FAILED_NODE_HOST="$2" - FAILED_NODE_PORT="$3" - FAILED_NODE_PGDATA="$4" - NEW_MASTER_NODE_ID="$5" - NEW_MASTER_NODE_HOST="$6" - OLD_MASTER_NODE_ID="$7" - OLD_PRIMARY_NODE_ID="$8" - NEW_MASTER_NODE_PORT="$9" - NEW_MASTER_NODE_PGDATA="${10}" - OLD_PRIMARY_NODE_HOST="${11}" - OLD_PRIMARY_NODE_PORT="${12}" - - PGHOME=/usr/pgsql-11 - - - logger -i -p local1.info failover.sh: start: failed_node_id=$FAILED_NODE_ID old_primary_node_id=$OLD_PRIMARY_NODE_ID failed_host=$FAILED_NODE_HOST new_master_host=$NEW_MASTER_NODE_HOST - - ## Test passwrodless SSH - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null - - if [ $? -ne 0 ]; then - logger -i -p local1.info failover.sh: passwrodless SSH to postgres@${NEW_MASTER_NODE_HOST} failed. Please setup passwrodless SSH. - exit 1 - fi - - ## If Standby node is down, skip failover. - if [ $FAILED_NODE_ID -ne $OLD_PRIMARY_NODE_ID ]; then - logger -i -p local1.info failover.sh: Standby node is down. Skipping failover. - - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@$OLD_PRIMARY_NODE_HOST -i ~/.ssh/id_rsa_pgpool " - ${PGHOME}/bin/psql -p $OLD_PRIMARY_NODE_PORT -c \"SELECT pg_drop_replication_slot('${FAILED_NODE_HOST}')\" - " - - if [ $? -ne 0 ]; then - logger -i -p local1.error failover.sh: drop replication slot "${FAILED_NODE_HOST}" failed - exit 1 - fi - - exit 0 - fi - - ## Promote Standby node. - logger -i -p local1.info failover.sh: Primary node is down, promote standby node ${NEW_MASTER_NODE_HOST}. - - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ${PGHOME}/bin/pg_ctl -D ${NEW_MASTER_NODE_PGDATA} -w promote - - if [ $? -ne 0 ]; then - logger -i -p local1.error failover.sh: new_master_host=$NEW_MASTER_NODE_HOST promote failed - exit 1 - fi - - logger -i -p local1.info failover.sh: end: new_master_node_id=$NEW_MASTER_NODE_ID started as the primary node - exit 0 +#!/bin/bash +# This script is run by failover_command. + +set -o xtrace +exec > >(logger -i -p local1.info) 2>&1 + +# Special values: +# %d = failed node id +# %h = failed node hostname +# %p = failed node port number +# %D = failed node database cluster path +# %m = new master node id +# %H = new master node hostname +# %M = old master node id +# %P = old primary node id +# %r = new master port number +# %R = new master database cluster path +# %N = old primary node hostname +# %S = old primary node port number +# %% = '%' character + +FAILED_NODE_ID="$1" +FAILED_NODE_HOST="$2" +FAILED_NODE_PORT="$3" +FAILED_NODE_PGDATA="$4" +NEW_MASTER_NODE_ID="$5" +NEW_MASTER_NODE_HOST="$6" +OLD_MASTER_NODE_ID="$7" +OLD_PRIMARY_NODE_ID="$8" +NEW_MASTER_NODE_PORT="$9" +NEW_MASTER_NODE_PGDATA="${10}" +OLD_PRIMARY_NODE_HOST="${11}" +OLD_PRIMARY_NODE_PORT="${12}" + +PGHOME=/usr/pgsql-11 + + +logger -i -p local1.info failover.sh: start: failed_node_id=$FAILED_NODE_ID old_primary_node_id=$OLD_PRIMARY_NODE_ID failed_host=$FAILED_NODE_HOST new_master_host=$NEW_MASTER_NODE_HOST + +## Test passwrodless SSH +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null + +if [ $? -ne 0 ]; then + logger -i -p local1.info failover.sh: passwrodless SSH to postgres@${NEW_MASTER_NODE_HOST} failed. Please setup passwrodless SSH. + exit 1 +fi + +## If Standby node is down, skip failover. +if [ $FAILED_NODE_ID -ne $OLD_PRIMARY_NODE_ID ]; then + logger -i -p local1.info failover.sh: Standby node is down. Skipping failover. + + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@$OLD_PRIMARY_NODE_HOST -i ~/.ssh/id_rsa_pgpool " + ${PGHOME}/bin/psql -p $OLD_PRIMARY_NODE_PORT -c \"SELECT pg_drop_replication_slot('${FAILED_NODE_HOST}')\" + " + + if [ $? -ne 0 ]; then + logger -i -p local1.error failover.sh: drop replication slot "${FAILED_NODE_HOST}" failed + exit 1 + fi + + exit 0 +fi + +## Promote Standby node. +logger -i -p local1.info failover.sh: Primary node is down, promote standby node ${NEW_MASTER_NODE_HOST}. + +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ${PGHOME}/bin/pg_ctl -D ${NEW_MASTER_NODE_PGDATA} -w promote + +if [ $? -ne 0 ]; then + logger -i -p local1.error failover.sh: new_master_host=$NEW_MASTER_NODE_HOST promote failed + exit 1 +fi + +logger -i -p local1.info failover.sh: end: new_master_node_id=$NEW_MASTER_NODE_ID started as the primary node +exit 0 @@ -572,176 +572,177 @@ /etc/pgpool-II/follow_master.sh - #!/bin/bash - # This script is run after failover_command to synchronize the Standby with the new Primary. - # First try pg_rewind. If pg_rewind failed, use pg_basebackup. - - set -o xtrace - exec > >(logger -i -p local1.info) 2>&1 - - # Special values: - # %d = failed node id - # %h = failed node hostname - # %p = failed node port number - # %D = failed node database cluster path - # %m = new master node id - # %H = new master node hostname - # %M = old master node id - # %P = old primary node id - # %r = new master port number - # %R = new master database cluster path - # %N = old primary node hostname - # %S = old primary node port number - # %% = '%' character - - FAILED_NODE_ID="$1" - FAILED_NODE_HOST="$2" - FAILED_NODE_PORT="$3" - FAILED_NODE_PGDATA="$4" - NEW_MASTER_NODE_ID="$5" - OLD_MASTER_NODE_ID="$6" - NEW_MASTER_NODE_HOST="$7" - OLD_PRIMARY_NODE_ID="$8" - NEW_MASTER_NODE_PORT="$9" - NEW_MASTER_NODE_PGDATA="${10}" - - PGHOME=/usr/pgsql-11 - ARCHIVEDIR=/var/lib/pgsql/archivedir - REPLUSER=repl - PCP_USER=pgpool - PGPOOL_PATH=/usr/bin - PCP_PORT=9898 - - logger -i -p local1.info follow_master.sh: start: Standby node ${FAILED_NODE_ID} - - ## Test passwrodless SSH - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null - - if [ $? -ne 0 ]; then - logger -i -p local1.info follow_master.sh: passwrodless SSH to postgres@${NEW_MASTER_NODE_HOST} failed. Please setup passwrodless SSH. - exit 1 - fi - - ## Get PostgreSQL major version - PGVERSION=`${PGHOME}/bin/initdb -V | awk '{print $3}' | sed 's/\..*//' | sed 's/\([0-9]*\)[a-zA-Z].*/\1/'` - - if [ $PGVERSION -ge 12 ]; then - RECOVERYCONF=${FAILED_NODE_PGDATA}/myrecovery.conf - else - RECOVERYCONF=${FAILED_NODE_PGDATA}/recovery.conf - fi - - ## Check the status of Standby - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ${PGHOME}/bin/pg_ctl -w -D ${FAILED_NODE_PGDATA} status - - - ## If Standby is running, synchronize it with the new Primary. - if [ $? -eq 0 ]; then - - logger -i -p local1.info follow_master.sh: pg_rewind for $FAILED_NODE_ID - - # Create replication slot "${FAILED_NODE_HOST}" - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " - ${PGHOME}/bin/psql -p ${NEW_MASTER_NODE_PORT} -c \"SELECT pg_create_physical_replication_slot('${FAILED_NODE_HOST}');\" - " - - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " - - set -o errexit - - ${PGHOME}/bin/pg_ctl -w -m f -D ${FAILED_NODE_PGDATA} stop - - cat > ${RECOVERYCONF} << EOT - primary_conninfo = 'host=${NEW_MASTER_NODE_HOST} port=${NEW_MASTER_NODE_PORT} user=${REPLUSER} application_name=${FAILED_NODE_HOST} passfile=''/var/lib/pgsql/.pgpass''' - recovery_target_timeline = 'latest' - restore_command = 'scp ${NEW_MASTER_NODE_HOST}:${ARCHIVEDIR}/%f %p' - primary_slot_name = '${FAILED_NODE_HOST}' - EOT - - if [ ${PGVERSION} -ge 12 ]; then - touch ${FAILED_NODE_PGDATA}/standby.signal - else - echo \"standby_mode = 'on'\" >> ${RECOVERYCONF} - fi - - ${PGHOME}/bin/pg_rewind -D ${FAILED_NODE_PGDATA} --source-server=\"user=postgres host=${NEW_MASTER_NODE_HOST} port=${NEW_MASTER_NODE_PORT}\" - - " - - if [ $? -ne 0 ]; then - logger -i -p local1.error follow_master.sh: end: pg_rewind failed. Try pg_basebackup. - - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " - - set -o errexit - - # Execute pg_basebackup - rm -rf ${FAILED_NODE_PGDATA} - rm -rf ${ARCHIVEDIR}/* - ${PGHOME}/bin/pg_basebackup -h ${NEW_MASTER_NODE_HOST} -U $REPLUSER -p ${NEW_MASTER_NODE_PORT} -D ${FAILED_NODE_PGDATA} -X stream - - if [ ${PGVERSION} -ge 12 ]; then - sed -i -e \"\\\$ainclude_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'\" \ - -e \"/^include_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'/d\" ${FAILED_NODE_PGDATA}/postgresql.conf - fi - - cat > ${RECOVERYCONF} << EOT - primary_conninfo = 'host=${NEW_MASTER_NODE_HOST} port=${NEW_MASTER_NODE_PORT} user=${REPLUSER} application_name=${FAILED_NODE_HOST} passfile=''/var/lib/pgsql/.pgpass''' - recovery_target_timeline = 'latest' - restore_command = 'scp ${NEW_MASTER_NODE_HOST}:${ARCHIVEDIR}/%f %p' - primary_slot_name = '${FAILED_NODE_HOST}' - EOT - - if [ ${PGVERSION} -ge 12 ]; then - touch ${FAILED_NODE_PGDATA}/standby.signal - else - echo \"standby_mode = 'on'\" >> ${RECOVERYCONF} - fi - " - - if [ $? -ne 0 ]; then - # drop replication slot - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " - ${PGHOME}/bin/psql -p ${NEW_MASTER_NODE_PORT} -c \"SELECT pg_drop_replication_slot('${FAILED_NODE_HOST}')\" - " - - logger -i -p local1.error follow_master.sh: end: pg_basebackup failed - exit 1 - fi - fi +#!/bin/bash +# This script is run after failover_command to synchronize the Standby with the new Primary. +# First try pg_rewind. If pg_rewind failed, use pg_basebackup. + +set -o xtrace +exec > >(logger -i -p local1.info) 2>&1 + +# Special values: +# %d = failed node id +# %h = failed node hostname +# %p = failed node port number +# %D = failed node database cluster path +# %m = new master node id +# %H = new master node hostname +# %M = old master node id +# %P = old primary node id +# %r = new master port number +# %R = new master database cluster path +# %N = old primary node hostname +# %S = old primary node port number +# %% = '%' character + +FAILED_NODE_ID="$1" +FAILED_NODE_HOST="$2" +FAILED_NODE_PORT="$3" +FAILED_NODE_PGDATA="$4" +NEW_MASTER_NODE_ID="$5" +OLD_MASTER_NODE_ID="$6" +NEW_MASTER_NODE_HOST="$7" +OLD_PRIMARY_NODE_ID="$8" +NEW_MASTER_NODE_PORT="$9" +NEW_MASTER_NODE_PGDATA="${10}" + +PGHOME=/usr/pgsql-11 +ARCHIVEDIR=/var/lib/pgsql/archivedir +REPLUSER=repl +PCP_USER=pgpool +PGPOOL_PATH=/usr/bin +PCP_PORT=9898 + +logger -i -p local1.info follow_master.sh: start: Standby node ${FAILED_NODE_ID} + +## Test passwrodless SSH +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null + +if [ $? -ne 0 ]; then + logger -i -p local1.info follow_master.sh: passwrodless SSH to postgres@${NEW_MASTER_NODE_HOST} failed. Please setup passwrodless SSH. + exit 1 +fi + +## Get PostgreSQL major version +PGVERSION=`${PGHOME}/bin/initdb -V | awk '{print $3}' | sed 's/\..*//' | sed 's/\([0-9]*\)[a-zA-Z].*/\1/'` + +if [ $PGVERSION -ge 12 ]; then +RECOVERYCONF=${FAILED_NODE_PGDATA}/myrecovery.conf +else +RECOVERYCONF=${FAILED_NODE_PGDATA}/recovery.conf +fi + +## Check the status of Standby +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ +postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ${PGHOME}/bin/pg_ctl -w -D ${FAILED_NODE_PGDATA} status + + +## If Standby is running, synchronize it with the new Primary. +if [ $? -eq 0 ]; then + + logger -i -p local1.info follow_master.sh: pg_rewind for $FAILED_NODE_ID + + # Create replication slot "${FAILED_NODE_HOST}" + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " + ${PGHOME}/bin/psql -p ${NEW_MASTER_NODE_PORT} -c \"SELECT pg_create_physical_replication_slot('${FAILED_NODE_HOST}');\" + " + + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " + + set -o errexit + + ${PGHOME}/bin/pg_ctl -w -m f -D ${FAILED_NODE_PGDATA} stop + + cat > ${RECOVERYCONF} << EOT +primary_conninfo = 'host=${NEW_MASTER_NODE_HOST} port=${NEW_MASTER_NODE_PORT} user=${REPLUSER} application_name=${FAILED_NODE_HOST} passfile=''/var/lib/pgsql/.pgpass''' +recovery_target_timeline = 'latest' +restore_command = 'scp ${NEW_MASTER_NODE_HOST}:${ARCHIVEDIR}/%f %p' +primary_slot_name = '${FAILED_NODE_HOST}' +EOT + + if [ ${PGVERSION} -ge 12 ]; then + touch ${FAILED_NODE_PGDATA}/standby.signal + else + echo \"standby_mode = 'on'\" >> ${RECOVERYCONF} + fi + + ${PGHOME}/bin/pg_rewind -D ${FAILED_NODE_PGDATA} --source-server=\"user=postgres host=${NEW_MASTER_NODE_HOST} port=${NEW_MASTER_NODE_PORT}\" + + " + + if [ $? -ne 0 ]; then + logger -i -p local1.error follow_master.sh: end: pg_rewind failed. Try pg_basebackup. + + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " + + set -o errexit + + # Execute pg_basebackup + rm -rf ${FAILED_NODE_PGDATA} + rm -rf ${ARCHIVEDIR}/* + ${PGHOME}/bin/pg_basebackup -h ${NEW_MASTER_NODE_HOST} -U $REPLUSER -p ${NEW_MASTER_NODE_PORT} -D ${FAILED_NODE_PGDATA} -X stream + + if [ ${PGVERSION} -ge 12 ]; then + sed -i -e \"\\\$ainclude_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'\" \ + -e \"/^include_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'/d\" ${FAILED_NODE_PGDATA}/postgresql.conf + fi + + cat > ${RECOVERYCONF} << EOT +primary_conninfo = 'host=${NEW_MASTER_NODE_HOST} port=${NEW_MASTER_NODE_PORT} user=${REPLUSER} application_name=${FAILED_NODE_HOST} passfile=''/var/lib/pgsql/.pgpass''' +recovery_target_timeline = 'latest' +restore_command = 'scp ${NEW_MASTER_NODE_HOST}:${ARCHIVEDIR}/%f %p' +primary_slot_name = '${FAILED_NODE_HOST}' +EOT + + if [ ${PGVERSION} -ge 12 ]; then + touch ${FAILED_NODE_PGDATA}/standby.signal + else + echo \"standby_mode = 'on'\" >> ${RECOVERYCONF} + fi + " + + if [ $? -ne 0 ]; then + # drop replication slot + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " + ${PGHOME}/bin/psql -p ${NEW_MASTER_NODE_PORT} -c \"SELECT pg_drop_replication_slot('${FAILED_NODE_HOST}')\" + " + + logger -i -p local1.error follow_master.sh: end: pg_basebackup failed + exit 1 + fi + fi - # start Standby node on ${FAILED_NODE_HOST} - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool $PGHOME/bin/pg_ctl -l /dev/null -w -D ${FAILED_NODE_PGDATA} start - - # If start Standby successfully, attach this node - if [ $? -eq 0 ]; then - - # Run pcp_attact_node to attach Standby node to Pgpool-II. - ${PGPOOL_PATH}/pcp_attach_node -w -h localhost -U $PCP_USER -p ${PCP_PORT} -n ${FAILED_NODE_ID} + # start Standby node on ${FAILED_NODE_HOST} + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool $PGHOME/bin/pg_ctl -l /dev/null -w -D ${FAILED_NODE_PGDATA} start + + # If start Standby successfully, attach this node + if [ $? -eq 0 ]; then + + # Run pcp_attact_node to attach Standby node to Pgpool-II. + ${PGPOOL_PATH}/pcp_attach_node -w -h localhost -U $PCP_USER -p ${PCP_PORT} -n ${FAILED_NODE_ID} - if [ $? -ne 0 ]; then - logger -i -p local1.error follow_master.sh: end: pcp_attach_node failed - exit 1 - fi - - # If start Standby failed, drop replication slot "${FAILED_NODE_HOST}" - else + if [ $? -ne 0 ]; then + logger -i -p local1.error follow_master.sh: end: pcp_attach_node failed + exit 1 + fi + + # If start Standby failed, drop replication slot "${FAILED_NODE_HOST}" + else - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool \ - ${PGHOME}/bin/psql -p ${NEW_MASTER_NODE_PORT} -c "SELECT pg_drop_replication_slot('${FAILED_NODE_HOST}')" - logger -i -p local1.error follow_master.sh: end: follow master command failed - exit 1 - fi - - else - logger -i -p local1.info follow_master.sh: failed_nod_id=${FAILED_NODE_ID} is not running. skipping follow master command - exit 0 - fi - - logger -i -p local1.info follow_master.sh: end: follow master command complete - exit 0 + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool \ + ${PGHOME}/bin/psql -p ${NEW_MASTER_NODE_PORT} -c "SELECT pg_drop_replication_slot('${FAILED_NODE_HOST}')" + + logger -i -p local1.error follow_master.sh: end: follow master command failed + exit 1 + fi + +else + logger -i -p local1.info follow_master.sh: failed_nod_id=${FAILED_NODE_ID} is not running. skipping follow master command + exit 0 +fi + +logger -i -p local1.info follow_master.sh: end: follow master command complete +exit 0 @@ -776,90 +777,90 @@ /var/lib/pgsql/11/data/recovery_1st_stage - #!/bin/bash - # This script is executed by "recovery_1st_stage" to recovery a Standby node. - - set -o xtrace - exec > >(logger -i -p local1.info) 2>&1 - - PRIMARY_NODE_PGDATA="$1" - DEST_NODE_HOST="$2" - DEST_NODE_PGDATA="$3" - PRIMARY_NODE_PORT="$4" - DEST_NODE_ID="$5" - DEST_NODE_PORT="$6" - - PRIMARY_NODE_HOST=$(hostname) - PGHOME=/usr/pgsql-11 - ARCHIVEDIR=/var/lib/pgsql/archivedir - REPLUSER=repl - - logger -i -p local1.info recovery_1st_stage: start: pg_basebackup for Standby node $DEST_NODE_ID - - ## Test passwrodless SSH - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${DEST_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null - - if [ $? -ne 0 ]; then - logger -i -p local1.info recovery_1st_stage: passwrodless SSH to postgres@${DEST_NODE_HOST} failed. Please setup passwrodless SSH. - exit 1 - fi - - ## Get PostgreSQL major version - PGVERSION=`${PGHOME}/bin/initdb -V | awk '{print $3}' | sed 's/\..*//' | sed 's/\([0-9]*\)[a-zA-Z].*/\1/'` - if [ $PGVERSION -ge 12 ]; then - RECOVERYCONF=${DEST_NODE_PGDATA}/myrecovery.conf - else - RECOVERYCONF=${DEST_NODE_PGDATA}/recovery.conf - fi - - ## Create replication slot "${DEST_NODE_HOST}" - ${PGHOME}/bin/psql -p ${PRIMARY_NODE_PORT} << EOQ - SELECT pg_create_physical_replication_slot('${DEST_NODE_HOST}'); - EOQ - - ## Execute pg_basebackup to recovery Standby node - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@$DEST_NODE_HOST -i ~/.ssh/id_rsa_pgpool " - - set -o errexit - - rm -rf $DEST_NODE_PGDATA - rm -rf $ARCHIVEDIR/* - - ${PGHOME}/bin/pg_basebackup -h $PRIMARY_NODE_HOST -U $REPLUSER -p $PRIMARY_NODE_PORT -D $DEST_NODE_PGDATA -X stream - - if [ ${PGVERSION} -ge 12 ]; then - sed -i -e \"\\\$ainclude_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'\" \ - -e \"/^include_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'/d\" ${DEST_NODE_PGDATA}/postgresql.conf - fi - - cat > ${RECOVERYCONF} << EOT - primary_conninfo = 'host=${PRIMARY_NODE_HOST} port=${PRIMARY_NODE_PORT} user=${REPLUSER} application_name=${DEST_NODE_HOST} passfile=''/var/lib/pgsql/.pgpass''' - recovery_target_timeline = 'latest' - restore_command = 'scp ${PRIMARY_NODE_HOST}:${ARCHIVEDIR}/%f %p' - primary_slot_name = '${DEST_NODE_HOST}' - EOT - - if [ ${PGVERSION} -ge 12 ]; then - touch ${DEST_NODE_PGDATA}/standby.signal - else - echo \"standby_mode = 'on'\" >> ${RECOVERYCONF} - fi - - sed -i \"s/#*port = .*/port = ${DEST_NODE_PORT}/\" ${DEST_NODE_PGDATA}/postgresql.conf - " - - if [ $? -ne 0 ]; then - - ${PGHOME}/bin/psql -p ${PRIMARY_NODE_PORT} << EOQ - SELECT pg_drop_replication_slot('${DEST_NODE_HOST}'); - EOQ - - logger -i -p local1.error recovery_1st_stage: end: pg_basebackup failed. online recovery failed - exit 1 - fi - - logger -i -p local1.info recovery_1st_stage: end: recovery_1st_stage complete - exit 0 +#!/bin/bash +# This script is executed by "recovery_1st_stage" to recovery a Standby node. + +set -o xtrace +exec > >(logger -i -p local1.info) 2>&1 + +PRIMARY_NODE_PGDATA="$1" +DEST_NODE_HOST="$2" +DEST_NODE_PGDATA="$3" +PRIMARY_NODE_PORT="$4" +DEST_NODE_ID="$5" +DEST_NODE_PORT="$6" + +PRIMARY_NODE_HOST=$(hostname) +PGHOME=/usr/pgsql-11 +ARCHIVEDIR=/var/lib/pgsql/archivedir +REPLUSER=repl + +logger -i -p local1.info recovery_1st_stage: start: pg_basebackup for Standby node $DEST_NODE_ID + +## Test passwrodless SSH +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${DEST_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null + +if [ $? -ne 0 ]; then + logger -i -p local1.info recovery_1st_stage: passwrodless SSH to postgres@${DEST_NODE_HOST} failed. Please setup passwrodless SSH. + exit 1 +fi + +## Get PostgreSQL major version +PGVERSION=`${PGHOME}/bin/initdb -V | awk '{print $3}' | sed 's/\..*//' | sed 's/\([0-9]*\)[a-zA-Z].*/\1/'` +if [ $PGVERSION -ge 12 ]; then + RECOVERYCONF=${DEST_NODE_PGDATA}/myrecovery.conf +else + RECOVERYCONF=${DEST_NODE_PGDATA}/recovery.conf +fi + +## Create replication slot "${DEST_NODE_HOST}" +${PGHOME}/bin/psql -p ${PRIMARY_NODE_PORT} << EOQ +SELECT pg_create_physical_replication_slot('${DEST_NODE_HOST}'); +EOQ + +## Execute pg_basebackup to recovery Standby node +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@$DEST_NODE_HOST -i ~/.ssh/id_rsa_pgpool " + + set -o errexit + + rm -rf $DEST_NODE_PGDATA + rm -rf $ARCHIVEDIR/* + + ${PGHOME}/bin/pg_basebackup -h $PRIMARY_NODE_HOST -U $REPLUSER -p $PRIMARY_NODE_PORT -D $DEST_NODE_PGDATA -X stream + + if [ ${PGVERSION} -ge 12 ]; then + sed -i -e \"\\\$ainclude_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'\" \ + -e \"/^include_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'/d\" ${DEST_NODE_PGDATA}/postgresql.conf + fi + + cat > ${RECOVERYCONF} << EOT +primary_conninfo = 'host=${PRIMARY_NODE_HOST} port=${PRIMARY_NODE_PORT} user=${REPLUSER} application_name=${DEST_NODE_HOST} passfile=''/var/lib/pgsql/.pgpass''' +recovery_target_timeline = 'latest' +restore_command = 'scp ${PRIMARY_NODE_HOST}:${ARCHIVEDIR}/%f %p' +primary_slot_name = '${DEST_NODE_HOST}' +EOT + + if [ ${PGVERSION} -ge 12 ]; then + touch ${DEST_NODE_PGDATA}/standby.signal + else + echo \"standby_mode = 'on'\" >> ${RECOVERYCONF} + fi + + sed -i \"s/#*port = .*/port = ${DEST_NODE_PORT}/\" ${DEST_NODE_PGDATA}/postgresql.conf +" + +if [ $? -ne 0 ]; then + + ${PGHOME}/bin/psql -p ${PRIMARY_NODE_PORT} << EOQ +SELECT pg_drop_replication_slot('${DEST_NODE_HOST}'); +EOQ + + logger -i -p local1.error recovery_1st_stage: end: pg_basebackup failed. online recovery failed + exit 1 +fi + +logger -i -p local1.info recovery_1st_stage: end: recovery_1st_stage complete +exit 0 @@ -868,39 +869,39 @@ /var/lib/pgsql/11/data/pgpool_remote_start - #!/bin/bash - # This script is run after recovery_1st_stage to start Standby node. +#!/bin/bash +# This script is run after recovery_1st_stage to start Standby node. - set -o xtrace - exec > >(logger -i -p local1.info) 2>&1 +set -o xtrace +exec > >(logger -i -p local1.info) 2>&1 - PGHOME=/usr/pgsql-11 - DEST_NODE_HOST="$1" - DEST_NODE_PGDATA="$2" +PGHOME=/usr/pgsql-11 +DEST_NODE_HOST="$1" +DEST_NODE_PGDATA="$2" - logger -i -p local1.info pgpool_remote_start: start: remote start Standby node $DEST_NODE_HOST +logger -i -p local1.info pgpool_remote_start: start: remote start Standby node $DEST_NODE_HOST - ## Test passwrodless SSH - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${DEST_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null +## Test passwrodless SSH +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${DEST_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null - if [ $? -ne 0 ]; then - logger -i -p local1.info pgpool_remote_start: passwrodless SSH to postgres@${DEST_NODE_HOST} failed. Please setup passwrodless SSH. - exit 1 - fi +if [ $? -ne 0 ]; then + logger -i -p local1.info pgpool_remote_start: passwrodless SSH to postgres@${DEST_NODE_HOST} failed. Please setup passwrodless SSH. + exit 1 +fi - ## Start Standby node - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@$DEST_NODE_HOST -i ~/.ssh/id_rsa_pgpool " - $PGHOME/bin/pg_ctl -l /dev/null -w -D $DEST_NODE_PGDATA start - " +## Start Standby node +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@$DEST_NODE_HOST -i ~/.ssh/id_rsa_pgpool " + $PGHOME/bin/pg_ctl -l /dev/null -w -D $DEST_NODE_PGDATA start +" - if [ $? -ne 0 ]; then - logger -i -p local1.error pgpool_remote_start: $DEST_NODE_HOST PostgreSQL start failed. - exit 1 - fi +if [ $? -ne 0 ]; then + logger -i -p local1.error pgpool_remote_start: $DEST_NODE_HOST PostgreSQL start failed. + exit 1 +fi - logger -i -p local1.info pgpool_remote_start: end: $DEST_NODE_HOST PostgreSQL started successfully. - exit 0 +logger -i -p local1.info pgpool_remote_start: end: $DEST_NODE_HOST PostgreSQL started successfully. +exit 0 diff --git a/doc/src/sgml/example-cluster.sgml b/doc/src/sgml/example-cluster.sgml index 351c43aa2..ea126dbec 100644 --- a/doc/src/sgml/example-cluster.sgml +++ b/doc/src/sgml/example-cluster.sgml @@ -515,82 +515,82 @@ /etc/pgpool-II/failover.sh - #!/bin/bash - # This script is run by failover_command. - - set -o xtrace - exec > >(logger -i -p local1.info) 2>&1 - - # Special values: - # %d = failed node id - # %h = failed node hostname - # %p = failed node port number - # %D = failed node database cluster path - # %m = new master node id - # %H = new master node hostname - # %M = old master node id - # %P = old primary node id - # %r = new master port number - # %R = new master database cluster path - # %N = old primary node hostname - # %S = old primary node port number - # %% = '%' character - - FAILED_NODE_ID="$1" - FAILED_NODE_HOST="$2" - FAILED_NODE_PORT="$3" - FAILED_NODE_PGDATA="$4" - NEW_MASTER_NODE_ID="$5" - NEW_MASTER_NODE_HOST="$6" - OLD_MASTER_NODE_ID="$7" - OLD_PRIMARY_NODE_ID="$8" - NEW_MASTER_NODE_PORT="$9" - NEW_MASTER_NODE_PGDATA="${10}" - OLD_PRIMARY_NODE_HOST="${11}" - OLD_PRIMARY_NODE_PORT="${12}" - - PGHOME=/usr/pgsql-11 - - - logger -i -p local1.info failover.sh: start: failed_node_id=$FAILED_NODE_ID old_primary_node_id=$OLD_PRIMARY_NODE_ID failed_host=$FAILED_NODE_HOST new_master_host=$NEW_MASTER_NODE_HOST - - ## Test passwrodless SSH - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null - - if [ $? -ne 0 ]; then - logger -i -p local1.info failover.sh: passwrodless SSH to postgres@${NEW_MASTER_NODE_HOST} failed. Please setup passwrodless SSH. - exit 1 - fi - - ## If Standby node is down, skip failover. - if [ $FAILED_NODE_ID -ne $OLD_PRIMARY_NODE_ID ]; then - logger -i -p local1.info failover.sh: Standby node is down. Skipping failover. - - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@$OLD_PRIMARY_NODE_HOST -i ~/.ssh/id_rsa_pgpool " - ${PGHOME}/bin/psql -p $OLD_PRIMARY_NODE_PORT -c \"SELECT pg_drop_replication_slot('${FAILED_NODE_HOST}')\" - " - - if [ $? -ne 0 ]; then - logger -i -p local1.error failover.sh: drop replication slot "${FAILED_NODE_HOST}" failed - exit 1 - fi - - exit 0 - fi - - ## Promote Standby node. - logger -i -p local1.info failover.sh: Primary node is down, promote standby node ${NEW_MASTER_NODE_HOST}. - - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ${PGHOME}/bin/pg_ctl -D ${NEW_MASTER_NODE_PGDATA} -w promote - - if [ $? -ne 0 ]; then - logger -i -p local1.error failover.sh: new_master_host=$NEW_MASTER_NODE_HOST promote failed - exit 1 - fi - - logger -i -p local1.info failover.sh: end: new_master_node_id=$NEW_MASTER_NODE_ID started as the primary node - exit 0 +#!/bin/bash +# This script is run by failover_command. + +set -o xtrace +exec > >(logger -i -p local1.info) 2>&1 + +# Special values: +# %d = failed node id +# %h = failed node hostname +# %p = failed node port number +# %D = failed node database cluster path +# %m = new master node id +# %H = new master node hostname +# %M = old master node id +# %P = old primary node id +# %r = new master port number +# %R = new master database cluster path +# %N = old primary node hostname +# %S = old primary node port number +# %% = '%' character + +FAILED_NODE_ID="$1" +FAILED_NODE_HOST="$2" +FAILED_NODE_PORT="$3" +FAILED_NODE_PGDATA="$4" +NEW_MASTER_NODE_ID="$5" +NEW_MASTER_NODE_HOST="$6" +OLD_MASTER_NODE_ID="$7" +OLD_PRIMARY_NODE_ID="$8" +NEW_MASTER_NODE_PORT="$9" +NEW_MASTER_NODE_PGDATA="${10}" +OLD_PRIMARY_NODE_HOST="${11}" +OLD_PRIMARY_NODE_PORT="${12}" + +PGHOME=/usr/pgsql-11 + + +logger -i -p local1.info failover.sh: start: failed_node_id=$FAILED_NODE_ID old_primary_node_id=$OLD_PRIMARY_NODE_ID failed_host=$FAILED_NODE_HOST new_master_host=$NEW_MASTER_NODE_HOST + +## Test passwrodless SSH +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null + +if [ $? -ne 0 ]; then + logger -i -p local1.info failover.sh: passwrodless SSH to postgres@${NEW_MASTER_NODE_HOST} failed. Please setup passwrodless SSH. + exit 1 +fi + +## If Standby node is down, skip failover. +if [ $FAILED_NODE_ID -ne $OLD_PRIMARY_NODE_ID ]; then + logger -i -p local1.info failover.sh: Standby node is down. Skipping failover. + + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@$OLD_PRIMARY_NODE_HOST -i ~/.ssh/id_rsa_pgpool " + ${PGHOME}/bin/psql -p $OLD_PRIMARY_NODE_PORT -c \"SELECT pg_drop_replication_slot('${FAILED_NODE_HOST}')\" + " + + if [ $? -ne 0 ]; then + logger -i -p local1.error failover.sh: drop replication slot "${FAILED_NODE_HOST}" failed + exit 1 + fi + + exit 0 +fi + +## Promote Standby node. +logger -i -p local1.info failover.sh: Primary node is down, promote standby node ${NEW_MASTER_NODE_HOST}. + +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ${PGHOME}/bin/pg_ctl -D ${NEW_MASTER_NODE_PGDATA} -w promote + +if [ $? -ne 0 ]; then + logger -i -p local1.error failover.sh: new_master_host=$NEW_MASTER_NODE_HOST promote failed + exit 1 +fi + +logger -i -p local1.info failover.sh: end: new_master_node_id=$NEW_MASTER_NODE_ID started as the primary node +exit 0 @@ -601,177 +601,177 @@ /etc/pgpool-II/follow_master.sh - #!/bin/bash - # This script is run after failover_command to synchronize the Standby with the new Primary. - # First try pg_rewind. If pg_rewind failed, use pg_basebackup. - - set -o xtrace - exec > >(logger -i -p local1.info) 2>&1 - - # Special values: - # %d = failed node id - # %h = failed node hostname - # %p = failed node port number - # %D = failed node database cluster path - # %m = new master node id - # %H = new master node hostname - # %M = old master node id - # %P = old primary node id - # %r = new master port number - # %R = new master database cluster path - # %N = old primary node hostname - # %S = old primary node port number - # %% = '%' character - - FAILED_NODE_ID="$1" - FAILED_NODE_HOST="$2" - FAILED_NODE_PORT="$3" - FAILED_NODE_PGDATA="$4" - NEW_MASTER_NODE_ID="$5" - OLD_MASTER_NODE_ID="$6" - NEW_MASTER_NODE_HOST="$7" - OLD_PRIMARY_NODE_ID="$8" - NEW_MASTER_NODE_PORT="$9" - NEW_MASTER_NODE_PGDATA="${10}" - - PGHOME=/usr/pgsql-11 - ARCHIVEDIR=/var/lib/pgsql/archivedir - REPLUSER=repl - PCP_USER=pgpool - PGPOOL_PATH=/usr/bin - PCP_PORT=9898 - - logger -i -p local1.info follow_master.sh: start: Standby node ${FAILED_NODE_ID} - - ## Test passwrodless SSH - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null - - if [ $? -ne 0 ]; then - logger -i -p local1.info follow_master.sh: passwrodless SSH to postgres@${NEW_MASTER_NODE_HOST} failed. Please setup passwrodless SSH. - exit 1 - fi - - ## Get PostgreSQL major version - PGVERSION=`${PGHOME}/bin/initdb -V | awk '{print $3}' | sed 's/\..*//' | sed 's/\([0-9]*\)[a-zA-Z].*/\1/'` - - if [ $PGVERSION -ge 12 ]; then - RECOVERYCONF=${FAILED_NODE_PGDATA}/myrecovery.conf - else - RECOVERYCONF=${FAILED_NODE_PGDATA}/recovery.conf - fi - - ## Check the status of Standby - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ${PGHOME}/bin/pg_ctl -w -D ${FAILED_NODE_PGDATA} status - - - ## If Standby is running, synchronize it with the new Primary. - if [ $? -eq 0 ]; then - - logger -i -p local1.info follow_master.sh: pg_rewind for $FAILED_NODE_ID - - # Create replication slot "${FAILED_NODE_HOST}" - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " - ${PGHOME}/bin/psql -p ${NEW_MASTER_NODE_PORT} -c \"SELECT pg_create_physical_replication_slot('${FAILED_NODE_HOST}');\" - " - - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " - - set -o errexit - - ${PGHOME}/bin/pg_ctl -w -m f -D ${FAILED_NODE_PGDATA} stop - - cat > ${RECOVERYCONF} << EOT - primary_conninfo = 'host=${NEW_MASTER_NODE_HOST} port=${NEW_MASTER_NODE_PORT} user=${REPLUSER} application_name=${FAILED_NODE_HOST} passfile=''/var/lib/pgsql/.pgpass''' - recovery_target_timeline = 'latest' - restore_command = 'scp ${NEW_MASTER_NODE_HOST}:${ARCHIVEDIR}/%f %p' - primary_slot_name = '${FAILED_NODE_HOST}' - EOT - - if [ ${PGVERSION} -ge 12 ]; then - touch ${FAILED_NODE_PGDATA}/standby.signal - else - echo \"standby_mode = 'on'\" >> ${RECOVERYCONF} - fi - - ${PGHOME}/bin/pg_rewind -D ${FAILED_NODE_PGDATA} --source-server=\"user=postgres host=${NEW_MASTER_NODE_HOST} port=${NEW_MASTER_NODE_PORT}\" - - " - - if [ $? -ne 0 ]; then - logger -i -p local1.error follow_master.sh: end: pg_rewind failed. Try pg_basebackup. - - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " - - set -o errexit - - # Execute pg_basebackup - rm -rf ${FAILED_NODE_PGDATA} - rm -rf ${ARCHIVEDIR}/* - ${PGHOME}/bin/pg_basebackup -h ${NEW_MASTER_NODE_HOST} -U $REPLUSER -p ${NEW_MASTER_NODE_PORT} -D ${FAILED_NODE_PGDATA} -X stream - - if [ ${PGVERSION} -ge 12 ]; then - sed -i -e \"\\\$ainclude_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'\" \ - -e \"/^include_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'/d\" ${FAILED_NODE_PGDATA}/postgresql.conf - fi - - cat > ${RECOVERYCONF} << EOT - primary_conninfo = 'host=${NEW_MASTER_NODE_HOST} port=${NEW_MASTER_NODE_PORT} user=${REPLUSER} application_name=${FAILED_NODE_HOST} passfile=''/var/lib/pgsql/.pgpass''' - recovery_target_timeline = 'latest' - restore_command = 'scp ${NEW_MASTER_NODE_HOST}:${ARCHIVEDIR}/%f %p' - primary_slot_name = '${FAILED_NODE_HOST}' - EOT - - if [ ${PGVERSION} -ge 12 ]; then - touch ${FAILED_NODE_PGDATA}/standby.signal - else - echo \"standby_mode = 'on'\" >> ${RECOVERYCONF} - fi - " - - if [ $? -ne 0 ]; then - # drop replication slot - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " - ${PGHOME}/bin/psql -p ${NEW_MASTER_NODE_PORT} -c \"SELECT pg_drop_replication_slot('${FAILED_NODE_HOST}')\" - " - - logger -i -p local1.error follow_master.sh: end: pg_basebackup failed - exit 1 - fi - fi +#!/bin/bash +# This script is run after failover_command to synchronize the Standby with the new Primary. +# First try pg_rewind. If pg_rewind failed, use pg_basebackup. + +set -o xtrace +exec > >(logger -i -p local1.info) 2>&1 + +# Special values: +# %d = failed node id +# %h = failed node hostname +# %p = failed node port number +# %D = failed node database cluster path +# %m = new master node id +# %H = new master node hostname +# %M = old master node id +# %P = old primary node id +# %r = new master port number +# %R = new master database cluster path +# %N = old primary node hostname +# %S = old primary node port number +# %% = '%' character + +FAILED_NODE_ID="$1" +FAILED_NODE_HOST="$2" +FAILED_NODE_PORT="$3" +FAILED_NODE_PGDATA="$4" +NEW_MASTER_NODE_ID="$5" +OLD_MASTER_NODE_ID="$6" +NEW_MASTER_NODE_HOST="$7" +OLD_PRIMARY_NODE_ID="$8" +NEW_MASTER_NODE_PORT="$9" +NEW_MASTER_NODE_PGDATA="${10}" + +PGHOME=/usr/pgsql-11 +ARCHIVEDIR=/var/lib/pgsql/archivedir +REPLUSER=repl +PCP_USER=pgpool +PGPOOL_PATH=/usr/bin +PCP_PORT=9898 + +logger -i -p local1.info follow_master.sh: start: Standby node ${FAILED_NODE_ID} + +## Test passwrodless SSH +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null + +if [ $? -ne 0 ]; then + logger -i -p local1.info follow_master.sh: passwrodless SSH to postgres@${NEW_MASTER_NODE_HOST} failed. Please setup passwrodless SSH. + exit 1 +fi + +## Get PostgreSQL major version +PGVERSION=`${PGHOME}/bin/initdb -V | awk '{print $3}' | sed 's/\..*//' | sed 's/\([0-9]*\)[a-zA-Z].*/\1/'` + +if [ $PGVERSION -ge 12 ]; then +RECOVERYCONF=${FAILED_NODE_PGDATA}/myrecovery.conf +else +RECOVERYCONF=${FAILED_NODE_PGDATA}/recovery.conf +fi + +## Check the status of Standby +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ +postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ${PGHOME}/bin/pg_ctl -w -D ${FAILED_NODE_PGDATA} status + + +## If Standby is running, synchronize it with the new Primary. +if [ $? -eq 0 ]; then + + logger -i -p local1.info follow_master.sh: pg_rewind for $FAILED_NODE_ID + + # Create replication slot "${FAILED_NODE_HOST}" + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " + ${PGHOME}/bin/psql -p ${NEW_MASTER_NODE_PORT} -c \"SELECT pg_create_physical_replication_slot('${FAILED_NODE_HOST}');\" + " + + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " + + set -o errexit + + ${PGHOME}/bin/pg_ctl -w -m f -D ${FAILED_NODE_PGDATA} stop + + cat > ${RECOVERYCONF} << EOT +primary_conninfo = 'host=${NEW_MASTER_NODE_HOST} port=${NEW_MASTER_NODE_PORT} user=${REPLUSER} application_name=${FAILED_NODE_HOST} passfile=''/var/lib/pgsql/.pgpass''' +recovery_target_timeline = 'latest' +restore_command = 'scp ${NEW_MASTER_NODE_HOST}:${ARCHIVEDIR}/%f %p' +primary_slot_name = '${FAILED_NODE_HOST}' +EOT + + if [ ${PGVERSION} -ge 12 ]; then + touch ${FAILED_NODE_PGDATA}/standby.signal + else + echo \"standby_mode = 'on'\" >> ${RECOVERYCONF} + fi + + ${PGHOME}/bin/pg_rewind -D ${FAILED_NODE_PGDATA} --source-server=\"user=postgres host=${NEW_MASTER_NODE_HOST} port=${NEW_MASTER_NODE_PORT}\" + + " + + if [ $? -ne 0 ]; then + logger -i -p local1.error follow_master.sh: end: pg_rewind failed. Try pg_basebackup. + + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " + + set -o errexit + + # Execute pg_basebackup + rm -rf ${FAILED_NODE_PGDATA} + rm -rf ${ARCHIVEDIR}/* + ${PGHOME}/bin/pg_basebackup -h ${NEW_MASTER_NODE_HOST} -U $REPLUSER -p ${NEW_MASTER_NODE_PORT} -D ${FAILED_NODE_PGDATA} -X stream + + if [ ${PGVERSION} -ge 12 ]; then + sed -i -e \"\\\$ainclude_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'\" \ + -e \"/^include_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'/d\" ${FAILED_NODE_PGDATA}/postgresql.conf + fi + + cat > ${RECOVERYCONF} << EOT +primary_conninfo = 'host=${NEW_MASTER_NODE_HOST} port=${NEW_MASTER_NODE_PORT} user=${REPLUSER} application_name=${FAILED_NODE_HOST} passfile=''/var/lib/pgsql/.pgpass''' +recovery_target_timeline = 'latest' +restore_command = 'scp ${NEW_MASTER_NODE_HOST}:${ARCHIVEDIR}/%f %p' +primary_slot_name = '${FAILED_NODE_HOST}' +EOT + + if [ ${PGVERSION} -ge 12 ]; then + touch ${FAILED_NODE_PGDATA}/standby.signal + else + echo \"standby_mode = 'on'\" >> ${RECOVERYCONF} + fi + " + + if [ $? -ne 0 ]; then + # drop replication slot + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool " + ${PGHOME}/bin/psql -p ${NEW_MASTER_NODE_PORT} -c \"SELECT pg_drop_replication_slot('${FAILED_NODE_HOST}')\" + " + + logger -i -p local1.error follow_master.sh: end: pg_basebackup failed + exit 1 + fi + fi - # start Standby node on ${FAILED_NODE_HOST} - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ - postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool $PGHOME/bin/pg_ctl -l /dev/null -w -D ${FAILED_NODE_PGDATA} start - - # If start Standby successfully, attach this node - if [ $? -eq 0 ]; then - - # Run pcp_attact_node to attach Standby node to Pgpool-II. - ${PGPOOL_PATH}/pcp_attach_node -w -h localhost -U $PCP_USER -p ${PCP_PORT} -n ${FAILED_NODE_ID} + # start Standby node on ${FAILED_NODE_HOST} + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + postgres@${FAILED_NODE_HOST} -i ~/.ssh/id_rsa_pgpool $PGHOME/bin/pg_ctl -l /dev/null -w -D ${FAILED_NODE_PGDATA} start + + # If start Standby successfully, attach this node + if [ $? -eq 0 ]; then + + # Run pcp_attact_node to attach Standby node to Pgpool-II. + ${PGPOOL_PATH}/pcp_attach_node -w -h localhost -U $PCP_USER -p ${PCP_PORT} -n ${FAILED_NODE_ID} - if [ $? -ne 0 ]; then - logger -i -p local1.error follow_master.sh: end: pcp_attach_node failed - exit 1 - fi - - # If start Standby failed, drop replication slot "${FAILED_NODE_HOST}" - else + if [ $? -ne 0 ]; then + logger -i -p local1.error follow_master.sh: end: pcp_attach_node failed + exit 1 + fi + + # If start Standby failed, drop replication slot "${FAILED_NODE_HOST}" + else - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool \ - ${PGHOME}/bin/psql -p ${NEW_MASTER_NODE_PORT} -c "SELECT pg_drop_replication_slot('${FAILED_NODE_HOST}')" - - logger -i -p local1.error follow_master.sh: end: follow master command failed - exit 1 - fi - - else - logger -i -p local1.info follow_master.sh: failed_nod_id=${FAILED_NODE_ID} is not running. skipping follow master command - exit 0 - fi - - logger -i -p local1.info follow_master.sh: end: follow master command complete - exit 0 + ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${NEW_MASTER_NODE_HOST} -i ~/.ssh/id_rsa_pgpool \ + ${PGHOME}/bin/psql -p ${NEW_MASTER_NODE_PORT} -c "SELECT pg_drop_replication_slot('${FAILED_NODE_HOST}')" + + logger -i -p local1.error follow_master.sh: end: follow master command failed + exit 1 + fi + +else + logger -i -p local1.info follow_master.sh: failed_nod_id=${FAILED_NODE_ID} is not running. skipping follow master command + exit 0 +fi + +logger -i -p local1.info follow_master.sh: end: follow master command complete +exit 0 @@ -811,90 +811,90 @@ /var/lib/pgsql/11/data/recovery_1st_stage - #!/bin/bash - # This script is executed by "recovery_1st_stage" to recovery a Standby node. - - set -o xtrace - exec > >(logger -i -p local1.info) 2>&1 - - PRIMARY_NODE_PGDATA="$1" - DEST_NODE_HOST="$2" - DEST_NODE_PGDATA="$3" - PRIMARY_NODE_PORT="$4" - DEST_NODE_ID="$5" - DEST_NODE_PORT="$6" - - PRIMARY_NODE_HOST=$(hostname) - PGHOME=/usr/pgsql-11 - ARCHIVEDIR=/var/lib/pgsql/archivedir - REPLUSER=repl - - logger -i -p local1.info recovery_1st_stage: start: pg_basebackup for Standby node $DEST_NODE_ID - - ## Test passwrodless SSH - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${DEST_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null - - if [ $? -ne 0 ]; then - logger -i -p local1.info recovery_1st_stage: passwrodless SSH to postgres@${DEST_NODE_HOST} failed. Please setup passwrodless SSH. - exit 1 - fi - - ## Get PostgreSQL major version - PGVERSION=`${PGHOME}/bin/initdb -V | awk '{print $3}' | sed 's/\..*//' | sed 's/\([0-9]*\)[a-zA-Z].*/\1/'` - if [ $PGVERSION -ge 12 ]; then - RECOVERYCONF=${DEST_NODE_PGDATA}/myrecovery.conf - else - RECOVERYCONF=${DEST_NODE_PGDATA}/recovery.conf - fi - - ## Create replication slot "${DEST_NODE_HOST}" - ${PGHOME}/bin/psql -p ${PRIMARY_NODE_PORT} << EOQ - SELECT pg_create_physical_replication_slot('${DEST_NODE_HOST}'); - EOQ - - ## Execute pg_basebackup to recovery Standby node - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@$DEST_NODE_HOST -i ~/.ssh/id_rsa_pgpool " - - set -o errexit - - rm -rf $DEST_NODE_PGDATA - rm -rf $ARCHIVEDIR/* - - ${PGHOME}/bin/pg_basebackup -h $PRIMARY_NODE_HOST -U $REPLUSER -p $PRIMARY_NODE_PORT -D $DEST_NODE_PGDATA -X stream - - if [ ${PGVERSION} -ge 12 ]; then - sed -i -e \"\\\$ainclude_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'\" \ - -e \"/^include_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'/d\" ${DEST_NODE_PGDATA}/postgresql.conf - fi - - cat > ${RECOVERYCONF} << EOT - primary_conninfo = 'host=${PRIMARY_NODE_HOST} port=${PRIMARY_NODE_PORT} user=${REPLUSER} application_name=${DEST_NODE_HOST} passfile=''/var/lib/pgsql/.pgpass''' - recovery_target_timeline = 'latest' - restore_command = 'scp ${PRIMARY_NODE_HOST}:${ARCHIVEDIR}/%f %p' - primary_slot_name = '${DEST_NODE_HOST}' - EOT - - if [ ${PGVERSION} -ge 12 ]; then - touch ${DEST_NODE_PGDATA}/standby.signal - else - echo \"standby_mode = 'on'\" >> ${RECOVERYCONF} - fi - - sed -i \"s/#*port = .*/port = ${DEST_NODE_PORT}/\" ${DEST_NODE_PGDATA}/postgresql.conf - " - - if [ $? -ne 0 ]; then - - ${PGHOME}/bin/psql -p ${PRIMARY_NODE_PORT} << EOQ - SELECT pg_drop_replication_slot('${DEST_NODE_HOST}'); - EOQ - - logger -i -p local1.error recovery_1st_stage: end: pg_basebackup failed. online recovery failed - exit 1 - fi - - logger -i -p local1.info recovery_1st_stage: end: recovery_1st_stage complete - exit 0 +#!/bin/bash +# This script is executed by "recovery_1st_stage" to recovery a Standby node. + +set -o xtrace +exec > >(logger -i -p local1.info) 2>&1 + +PRIMARY_NODE_PGDATA="$1" +DEST_NODE_HOST="$2" +DEST_NODE_PGDATA="$3" +PRIMARY_NODE_PORT="$4" +DEST_NODE_ID="$5" +DEST_NODE_PORT="$6" + +PRIMARY_NODE_HOST=$(hostname) +PGHOME=/usr/pgsql-11 +ARCHIVEDIR=/var/lib/pgsql/archivedir +REPLUSER=repl + +logger -i -p local1.info recovery_1st_stage: start: pg_basebackup for Standby node $DEST_NODE_ID + +## Test passwrodless SSH +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${DEST_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null + +if [ $? -ne 0 ]; then + logger -i -p local1.info recovery_1st_stage: passwrodless SSH to postgres@${DEST_NODE_HOST} failed. Please setup passwrodless SSH. + exit 1 +fi + +## Get PostgreSQL major version +PGVERSION=`${PGHOME}/bin/initdb -V | awk '{print $3}' | sed 's/\..*//' | sed 's/\([0-9]*\)[a-zA-Z].*/\1/'` +if [ $PGVERSION -ge 12 ]; then + RECOVERYCONF=${DEST_NODE_PGDATA}/myrecovery.conf +else + RECOVERYCONF=${DEST_NODE_PGDATA}/recovery.conf +fi + +## Create replication slot "${DEST_NODE_HOST}" +${PGHOME}/bin/psql -p ${PRIMARY_NODE_PORT} << EOQ +SELECT pg_create_physical_replication_slot('${DEST_NODE_HOST}'); +EOQ + +## Execute pg_basebackup to recovery Standby node +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@$DEST_NODE_HOST -i ~/.ssh/id_rsa_pgpool " + + set -o errexit + + rm -rf $DEST_NODE_PGDATA + rm -rf $ARCHIVEDIR/* + + ${PGHOME}/bin/pg_basebackup -h $PRIMARY_NODE_HOST -U $REPLUSER -p $PRIMARY_NODE_PORT -D $DEST_NODE_PGDATA -X stream + + if [ ${PGVERSION} -ge 12 ]; then + sed -i -e \"\\\$ainclude_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'\" \ + -e \"/^include_if_exists = '$(echo ${RECOVERYCONF} | sed -e 's/\//\\\//g')'/d\" ${DEST_NODE_PGDATA}/postgresql.conf + fi + + cat > ${RECOVERYCONF} << EOT +primary_conninfo = 'host=${PRIMARY_NODE_HOST} port=${PRIMARY_NODE_PORT} user=${REPLUSER} application_name=${DEST_NODE_HOST} passfile=''/var/lib/pgsql/.pgpass''' +recovery_target_timeline = 'latest' +restore_command = 'scp ${PRIMARY_NODE_HOST}:${ARCHIVEDIR}/%f %p' +primary_slot_name = '${DEST_NODE_HOST}' +EOT + + if [ ${PGVERSION} -ge 12 ]; then + touch ${DEST_NODE_PGDATA}/standby.signal + else + echo \"standby_mode = 'on'\" >> ${RECOVERYCONF} + fi + + sed -i \"s/#*port = .*/port = ${DEST_NODE_PORT}/\" ${DEST_NODE_PGDATA}/postgresql.conf +" + +if [ $? -ne 0 ]; then + + ${PGHOME}/bin/psql -p ${PRIMARY_NODE_PORT} << EOQ +SELECT pg_drop_replication_slot('${DEST_NODE_HOST}'); +EOQ + + logger -i -p local1.error recovery_1st_stage: end: pg_basebackup failed. online recovery failed + exit 1 +fi + +logger -i -p local1.info recovery_1st_stage: end: recovery_1st_stage complete +exit 0 @@ -903,39 +903,39 @@ /var/lib/pgsql/11/data/pgpool_remote_start - #!/bin/bash - # This script is run after recovery_1st_stage to start Standby node. +#!/bin/bash +# This script is run after recovery_1st_stage to start Standby node. - set -o xtrace - exec > >(logger -i -p local1.info) 2>&1 +set -o xtrace +exec > >(logger -i -p local1.info) 2>&1 - PGHOME=/usr/pgsql-11 - DEST_NODE_HOST="$1" - DEST_NODE_PGDATA="$2" +PGHOME=/usr/pgsql-11 +DEST_NODE_HOST="$1" +DEST_NODE_PGDATA="$2" - logger -i -p local1.info pgpool_remote_start: start: remote start Standby node $DEST_NODE_HOST +logger -i -p local1.info pgpool_remote_start: start: remote start Standby node $DEST_NODE_HOST - ## Test passwrodless SSH - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${DEST_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null +## Test passwrodless SSH +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@${DEST_NODE_HOST} -i ~/.ssh/id_rsa_pgpool ls /tmp > /dev/null - if [ $? -ne 0 ]; then - logger -i -p local1.info pgpool_remote_start: passwrodless SSH to postgres@${DEST_NODE_HOST} failed. Please setup passwrodless SSH. - exit 1 - fi +if [ $? -ne 0 ]; then + logger -i -p local1.info pgpool_remote_start: passwrodless SSH to postgres@${DEST_NODE_HOST} failed. Please setup passwrodless SSH. + exit 1 +fi - ## Start Standby node - ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@$DEST_NODE_HOST -i ~/.ssh/id_rsa_pgpool " - $PGHOME/bin/pg_ctl -l /dev/null -w -D $DEST_NODE_PGDATA start - " +## Start Standby node +ssh -T -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null postgres@$DEST_NODE_HOST -i ~/.ssh/id_rsa_pgpool " + $PGHOME/bin/pg_ctl -l /dev/null -w -D $DEST_NODE_PGDATA start +" - if [ $? -ne 0 ]; then - logger -i -p local1.error pgpool_remote_start: $DEST_NODE_HOST PostgreSQL start failed. - exit 1 - fi +if [ $? -ne 0 ]; then + logger -i -p local1.error pgpool_remote_start: $DEST_NODE_HOST PostgreSQL start failed. + exit 1 +fi - logger -i -p local1.info pgpool_remote_start: end: $DEST_NODE_HOST PostgreSQL started successfully. - exit 0 +logger -i -p local1.info pgpool_remote_start: end: $DEST_NODE_HOST PostgreSQL started successfully. +exit 0