From 34978e07455b07294e62dbf76836c913d712a81f Mon Sep 17 00:00:00 2001 From: =?utf8?q?Felix=20D=C3=B6rre?= Date: Tue, 14 Nov 2017 23:53:55 +0100 Subject: [PATCH] add: backup mechanism for postgresql Change-Id: I65c9cd6c52b7d539fbc7dc59cc7436b64186dd73 --- backup | 109 ++++++++++++++ .../production/manifests/postgres-primary.pp | 23 ++- manager/backup | 138 ++++++++++++++++++ manager/config | 10 ++ manager/setup | 6 +- 5 files changed, 280 insertions(+), 6 deletions(-) create mode 100755 backup create mode 100755 manager/backup diff --git a/backup b/backup new file mode 100755 index 0000000..e1be93e --- /dev/null +++ b/backup @@ -0,0 +1,109 @@ +#!/bin/bash +printf "Backup service ready\n" +postgres_cluster=9.6/main +if ! which chunked >/dev/null; then + printf "Installing chunked en-/decoder\n" >&2 + # Install signing key for debian repo 'deb2.dogcraft.de' where 'chunked' is hosted. Downloadable from "http://deb.dogcraft.de/signer.gpg". + apt-key add - >&2 <&2 + apt-get update >&2 + apt-get install -y chunked >&2 +fi + +while :; do + read -r command || break + if [[ $command == base ]]; then + sudo lxc-attach -n postgres-primary -- su -c 'pg_basebackup -Ft -z -D - -P' postgres | chunked + printf "base backup done\n" + elif [[ $command == incremental ]]; then + read -e req_name + read -e req_hash + if ! grep -qi -- '^[A-Z0-9]\{24\}$' <<< $req_name; then + printf "Error: invalid WAL-name.\n" >&2 + printf "Error\n" + fi + real_hash="$(sha256sum "/data/postgres/data/archive/$req_name" | cut -d" " -f1)" + if [[ $req_hash != "-" ]] && [[ $real_hash != $req_hash ]]; then + printf "Error: hash mismatch on expected %s != provided %s\n" "$real_hash" "$req_hash" >&2 + printf "Error\n" + exit 1 + fi + printf "Ready\n" + + files=( /data/postgres/data/archive/* ) + for i in "${files[@]}"; do + name="$(basename $i)" + if ( [[ "$name" != "$req_name".* ]] && [[ $name > $req_name ]] ) || ( ( [[ "$name" == "$req_name".* ]] || [[ $req_name == $name ]] ) && [[ $req_hash == "-" ]] ); then + printf "%s\n" "$name" + fi + done | tar cz -C /data/postgres/data/archive -T - | chunked + printf "incremental backup done, confirm cleanup!\n" + read -e confirmation + if [[ $confirmation == "y" ]]; then + printf "Cleaning up archive\n" >&2 + lxc-attach -n postgres-primary -- pg_archivecleanup /var/lib/postgresql/archive/ "$req_name" + else + printf "Not doing cleanup\n" >&2 + fi + elif [[ $command == restore ]]; then + # for now (and quick development) we override + rm -R /data/postgres/data + + if [[ -d /data/postgres/data ]]; then + printf "error\n" + exit 1 + fi + printf "postgres base\n" + mkdir -p "/data/postgres/data/${postgres_cluster}" + chunked decode > /data/postgres/data/pg_base.tar.gz + mkdir -p /data/postgres/data/restore + while :; do + printf "incremental?\n" + read -e inc + if [[ $inc != "y" ]]; then + break + fi + chunked decode | tar xvz -C /data/postgres/data/restore >&2 + done + cat > "/data/postgres/data/${postgres_cluster}/recovery.conf" < "/data/postgres/conf/${postgres_cluster}/start.conf" + touch "/data/postgres/conf/${postgres_cluster}/postgresql.conf" + elif [[ $command == end ]]; then + printf "end\n" + fi +done diff --git a/environments/production/manifests/postgres-primary.pp b/environments/production/manifests/postgres-primary.pp index c1f015a..9d28846 100644 --- a/environments/production/manifests/postgres-primary.pp +++ b/environments/production/manifests/postgres-primary.pp @@ -2,17 +2,31 @@ node postgres-primary { include container::contained include container::no_ssh + exec { 'backup installed': + before => Package['postgresql'], + notify => Exec['backup permissions corrected'], + command => '! [ -f /var/lib/postgresql/9.6/main/PG_VERSION ] && mkdir -p /var/lib/postgresql/9.6/main && tar xzf /var/lib/postgresql/pg_base.tar.gz -C /var/lib/postgresql/9.6/main', + onlyif => '[ -f /var/lib/postgresql/pg_base.tar.gz ]', + provider => 'shell' + } package{ 'postgresql': ensure => 'installed', install_options => ['--no-install-recommends'], - } - + }-> class { 'postgresql::globals': version => '9.6', }-> class { 'postgresql::server': listen_addresses => '*', - } -> + } + exec { 'backup permissions corrected': + require => Class['postgresql::server::install'], + before => Class['postgresql::server::initdb'], + command => 'chown -R postgres:postgres /var/lib/postgresql && rm /var/lib/postgresql/pg_base.tar.gz', + onlyif => '[ -f /var/lib/postgresql/pg_base.tar.gz ]', + refreshonly => 'true', + provider => 'shell' + } postgresql::server::db { 'gigi': require => Package['postgresql'], user => 'gigi', @@ -30,7 +44,7 @@ node postgres-primary { } postgresql::server::db { 'quiz': - require => Class['postgresql::server'], + require => Exec['backup installed'], user => 'quiz', password => postgresql_password('quiz', $passwords[postgres][quiz]), } @@ -55,6 +69,7 @@ node postgres-primary { value => 'on' } file{'/var/lib/postgresql/archive/': + require => Exec['backup permissions corrected'], ensure => 'directory', owner => 'postgres' } -> diff --git a/manager/backup b/manager/backup new file mode 100755 index 0000000..22d536c --- /dev/null +++ b/manager/backup @@ -0,0 +1,138 @@ +#!/bin/bash + +targetHost=$1 +targetHost=${targetHost%/} +source config +source "$targetHost/config" + +if ! which chunked >/dev/null; then + printf "Requires 'chunked' package from deb2.dogcraft.de\n" >&2 + exit 1 +fi + + + +backup_target="$targetHost/backups" +coproc { + ssh_target "sudo ./backup" + read -r end +} +read -r line <&${COPROC[0]} || exit 1; +if [[ $line != "Backup service ready" ]]; then + echo "Backup service did not respond" + exit 1 +fi +function start_backup { + ident="$(date "+%Y-%m-%d-%H-%M-%S")" + self="$backup_target/$ident" + mkdir -p "$self" + if [[ -h "$backup_target/last" ]]; then + last="$(readlink "$backup_target/last")" + ln -s "../$last" "$self/prev" + rm "$backup_target/last" + fi + ln -s "$ident" "$backup_target/last" +} +function base { + printf "base\n" >&${COPROC[1]} + chunked decode <&${COPROC[0]} > "$self/pg_base.tar.gz" + ls -alsh -- "$self/pg_base.tar.gz" + read -r line <&${COPROC[0]} || exit 1; + echo "pg_base done: $line" + echo "Backup info: " + tar xzO backup_label < "$self/pg_base.tar.gz" +} +function incremental { + last= + hash= + if [[ $(find -L "$self" -maxdepth 14 -name "pg_base.tar.gz" | wc -l) -lt 1 ]]; then + printf "doing pg_base backup to $self/pg_base.tar.gz:\n" + base + last="$(tar xzO backup_label < "$self/pg_base.tar.gz" | grep "^START WAL LOCATION: " | sed "s/.*(file \\(.*\\))/\\1/")" + hash="-" + else + last="" + folder="$self" + while [[ $last == "" ]]; do + folder="$folder/prev" + last="$(tar tf "$folder/pg_wal.tar.gz" | grep "^[A-Z0-9]*$" | tail -n 1)" + done + echo "Found last WAL file in backup: $folder" + hash=$(tar xfO "$folder/pg_wal.tar.gz" "$last" | sha256sum | cut -d" " -f1) + fi + printf "Last WAL-name: %s\n" "$last" + printf "incremental\n" >&${COPROC[1]} + printf "%s\n" "$last" >&${COPROC[1]} + printf "%s\n" "$hash" >&${COPROC[1]} + read -r line <&${COPROC[0]} || exit 1; + if [[ "$line" != "Ready" ]]; then + printf "incremental backup didn't start\n" + exit 1 + fi + chunked decode <&${COPROC[0]} > "$self/pg_wal.tar.gz" + printf "Tar contents\n" + tar tf "$self/pg_wal.tar.gz" + ls -alsh -- "$self/pg_wal.tar.gz" + read -r line <&${COPROC[0]} || exit 1; + if [[ $line == "incremental backup done, confirm cleanup!" ]]; then + printf "y\n" >&${COPROC[1]} + else + printf "Done, but got strange line for cleanup confirmation: %s\n" "$line" + fi + printf "Done incremental backup\n" +} + +if [[ "$2" == "restore" ]]; then + sourceHost=$3 + sourceHost=${sourceHost%/} + backup_target="${sourceHost}/backups" + folder="$backup_target/last" + printf "Restoring backup %s\n" "$(readlink -e -- "$folder")" + tar tf "$folder/pg_wal.tar.gz" + while ! [[ -f "$folder/pg_base.tar.gz" ]]; do + folder="$folder/prev" + printf "Requires backup %s\n" "$(readlink -e -- "$folder")" + tar tf "$folder/pg_wal.tar.gz" + done + #tar tf "$folder/pg_base.tar.gz" + printf "restore\n" >&${COPROC[1]} + read -r reply <&${COPROC[0]} || exit 1; + if [[ $reply != "postgres base" ]]; then + printf "Service is not ready to receive backup: %s\n" "$reply" + exit 1 + fi + echo "sending base" + chunked < "$folder/pg_base.tar.gz" >&${COPROC[1]} + echo "done sending base" + folder="$backup_target/last" + while :; do + read -r reply <&${COPROC[0]} || exit 1; + if [[ $reply != "incremental?" ]]; then + printf "Service is not ready to receive backup: %s\n" "$reply" + exit 1 + fi + printf "y\n" >&${COPROC[1]} + printf "Sending pg_wal from %s\n" "$(readlink -e -- "$folder")" + chunked < "$folder/pg_wal.tar.gz" >&${COPROC[1]} + if [[ -f "$folder/pg_base.tar.gz" ]]; then + break + fi + folder="$folder/prev" + done + read -r reply <&${COPROC[0]} || exit 1; + if [[ $reply != "incremental?" ]]; then + printf "Service is not ready to receive backup: %s\n" "$reply" + exit 1 + fi + printf "n\n" >&${COPROC[1]} +elif [[ "$2" == "backup" ]]; then + start_backup + incremental +else + printf "Error, unknown sub command: %s\n" "$2" >&2 +fi + + +printf "end\n" >&${COPROC[1]} +read -r line <&${COPROC[0]} || exit 1; +printf "END: %s\n" "$line" diff --git a/manager/config b/manager/config index e0e6753..3d7b25a 100755 --- a/manager/config +++ b/manager/config @@ -86,3 +86,13 @@ function mcurl { function admin_ssh { ssh -i admin-key -p 2222 "admin@$to" "$@" } + +# Install backup is a hook to be triggered before the sytem is set up +function install_backup { + : +} + +function populate_system { + # default is create fresh data using bootstrap-user + execute-bootstrap-user +} diff --git a/manager/setup b/manager/setup index 0fb4061..88e4516 100755 --- a/manager/setup +++ b/manager/setup @@ -135,12 +135,14 @@ if [[ "$2" == "update" ]]; then fi configure ensure_nre + +install_backup + title 'S3: puppet-1' ssh_target -t 'sudo ./bootstrap' title 'S3: puppet-2' ssh_target -t 'sudo ./bootstrap' -[[ -f ../../migrate ]] && ( cd ../.. && bash migrate "$targetHost") if [[ -f tricks ]]; then cat tricks | ssh_target 'cat > tricks && chmod +x tricks' ssh_target -t 'bash tricks' @@ -151,7 +153,7 @@ if [[ $signerLocation == "self" ]] && [[ $(ssh_target 'ps -ef | grep tcpseria[l] exit 1 fi -execute-bootstrap-user +populate_system title 'bash' eval $(ssh-agent -k) -- 2.39.2