manager/backup

   1 #!/bin/bash
   2 targetHost=$1
   3 targetHost=${targetHost%/}
   4 source config
   5 source "$targetHost/config"
   6
   7 if ! which chunked >/dev/null; then
   8     printf "Requires 'chunked' package from deb2.dogcraft.de\n" >&2
   9     exit 1
  10 fi
  11
  12
  13
  14 backup_target="$targetHost/backups"
  15 coproc {
  16     ssh_target "sudo ./backup"
  17     read -r end
  18 }
  19 read -r line <&${COPROC[0]} || exit 1;
  20 if [[ $line != "Backup service ready" ]]; then
  21     echo "Backup service did not respond"
  22     exit 1
  23 fi
  24 function start_backup {
  25     ident="$(date "+%Y-%m-%d-%H-%M-%S")"
  26     self="$backup_target/$ident"
  27     mkdir -p "$self"
  28     if [[ -h "$backup_target/last" ]]; then
  29         last="$(readlink "$backup_target/last")"
  30         ln -s "../$last" "$self/prev"
  31         rm "$backup_target/last"
  32     fi
  33     ln -s "$ident" "$backup_target/last"
  34 }
  35 function base {
  36     printf "base\n" >&${COPROC[1]}
  37     chunked decode <&${COPROC[0]} > "$self/pg_base.tar.gz"
  38     ls -alsh -- "$self/pg_base.tar.gz"
  39     read -r line <&${COPROC[0]} || exit 1;
  40     echo "pg_base done: $line"
  41     echo "Backup info: "
  42     tar xzO backup_label <  "$self/pg_base.tar.gz"
  43 }
  44 function incremental {
  45     last=
  46     hash=
  47     if [[ $(find -L "$self" -maxdepth 14 -name "pg_base.tar.gz" | wc -l) -lt 1 ]]; then
  48         printf "doing pg_base backup to $self/pg_base.tar.gz:\n"
  49         base
  50         last="$(tar xzO backup_label <  "$self/pg_base.tar.gz" | grep "^START WAL LOCATION: " | sed "s/.*(file \\(.*\\))/\\1/")"
  51         hash="-"
  52     else
  53         last=""
  54         folder="$self"
  55         while [[ $last == "" ]]; do
  56             folder="$folder/prev"
  57             last="$(tar tf "$folder/pg_wal.tar.gz" | grep "^[A-Z0-9]*$" | tail -n 1)"
  58         done
  59         echo "Found last WAL file in backup: $folder"
  60         hash=$(tar xfO "$folder/pg_wal.tar.gz" "$last" | sha256sum | cut -d" " -f1)
  61     fi
  62     printf "Last WAL-name: %s\n" "$last"
  63     printf "incremental\n" >&${COPROC[1]}
  64     printf "%s\n" "$last" >&${COPROC[1]}
  65     printf "%s\n" "$hash" >&${COPROC[1]}
  66     read -r line <&${COPROC[0]} || exit 1;
  67     if [[ "$line" != "Ready" ]]; then
  68         printf "incremental backup didn't start\n"
  69         exit 1
  70     fi
  71     chunked decode <&${COPROC[0]} > "$self/pg_wal.tar.gz"
  72     printf "Tar contents\n"
  73     tar tf "$self/pg_wal.tar.gz"
  74     ls -alsh -- "$self/pg_wal.tar.gz"
  75     read -r line <&${COPROC[0]} || exit 1;
  76     if [[ $line == "incremental backup done, confirm cleanup!" ]]; then
  77         printf "y\n" >&${COPROC[1]}
  78     else
  79         printf "Done, but got strange line for cleanup confirmation: %s\n" "$line"
  80     fi
  81     printf "Done incremental backup\n"
  82 }
  83
  84 # This code snippet cannot be used inline and therefore has to be a function.
  85 # When calling this function like 'receive_journal "journal" <&$fd', the
  86 # file-descriptor "$fd" is resolved in the calling original shell.
  87 # However when inlining this code like "... | ... <&$fd" the file-descriptor
  88 # is resolved in the first subshell created by this command and therefore
  89 # not valid anymore.
  90 function receive_journal {
  91     chunked decode | /lib/systemd/systemd-journal-remote -o "$1" -
  92 }
  93 function journal {
  94     printf "Fetching journals...\n"
  95     printf "journal\n" >&${COPROC[1]}
  96     folder="$self"
  97     for i in {1..10}; do
  98         folder="$folder/prev"
  99         if [[ -f "$folder/journal-until" ]]; then
 100             break;
 101         fi
 102     done
 103     if [[ -f "$folder/journal-until" ]]; then
 104         printf "From: "
 105         from="$(cat "$folder/journal-until")"
 106         date --utc -d "@$from"
 107         printf -- "%s\n" "$from" >&${COPROC[1]}
 108     else
 109         printf "From: start\n"
 110         printf -- "-\n" >&${COPROC[1]}
 111     fi
 112     read -r until <&${COPROC[0]} || exit 1;
 113     if [[ $until == "no journals" ]]; then
 114         printf "no journal events\n"
 115         return 1
 116     fi
 117     if [[ ! $until == "Until: "* ]]; then
 118         printf "Unexpected Until line: %s\n" "$until"
 119         exit 1
 120     fi
 121     until="${until#Until: }"
 122     printf "until: "
 123     date --utc -d "@$until"
 124
 125     printf "Until: %s\n" "$until"
 126     while :; do
 127         read -r line <&${COPROC[0]} || exit 1;
 128         if [[ $line == "end-of-journals" ]]; then
 129             break;
 130         fi
 131         jnl="${line#journal: }"
 132         printf "journal: %s\n" "$jnl"
 133         receive_journal "$self/part-$jnl.journal" <&${COPROC[0]}
 134     done
 135     journalctl --file="$self/part-*" -o export | /lib/systemd/systemd-journal-remote -o "$self/all.journal" -
 136     printf "Removing split journals:\n"
 137     rm -v "$self/part-"*.journal
 138     cat > "$self/journal-until" <<< "$until"
 139     ls -als "$self"
 140 }
 141
 142 if [[ "$2" == "restore" ]]; then
 143     sourceHost=$3
 144     sourceHost=${sourceHost%/}
 145     backup_target="${sourceHost}/backups"
 146     folder="$backup_target/last"
 147     printf "Restoring backup %s\n" "$(readlink -e -- "$folder")"
 148     tar tf "$folder/pg_wal.tar.gz"
 149     while ! [[ -f "$folder/pg_base.tar.gz" ]]; do
 150         folder="$folder/prev"
 151         printf "Requires backup %s\n" "$(readlink -e -- "$folder")"
 152         tar tf "$folder/pg_wal.tar.gz"
 153     done
 154     #tar tf "$folder/pg_base.tar.gz"
 155     printf "restore\n" >&${COPROC[1]}
 156     read -r reply <&${COPROC[0]} || exit 1;
 157     if [[ $reply != "postgres base" ]]; then
 158         printf "Service is not ready to receive backup: %s\n" "$reply"
 159         exit 1
 160     fi
 161     echo "sending base"
 162     chunked < "$folder/pg_base.tar.gz" >&${COPROC[1]}
 163     echo "done sending base"
 164     folder="$backup_target/last"
 165     while :; do
 166         read -r reply <&${COPROC[0]} || exit 1;
 167         if [[ $reply != "incremental?" ]]; then
 168             printf "Service is not ready to receive backup: %s\n" "$reply"
 169             exit 1
 170         fi
 171         printf "y\n" >&${COPROC[1]}
 172         printf "Sending pg_wal from %s\n" "$(readlink -e -- "$folder")"
 173         chunked < "$folder/pg_wal.tar.gz" >&${COPROC[1]}
 174         if [[ -f "$folder/pg_base.tar.gz" ]]; then
 175             break
 176         fi
 177         folder="$folder/prev"
 178     done
 179     read -r reply <&${COPROC[0]} || exit 1;
 180     if [[ $reply != "incremental?" ]]; then
 181         printf "Service is not ready to receive backup: %s\n" "$reply"
 182         exit 1
 183     fi
 184     printf "n\n" >&${COPROC[1]}
 185 elif [[ "$2" == "backup" ]]; then
 186     if ! [[ -x /lib/systemd/systemd-journal-remote ]]; then
 187         printf "This script requires 'systemd-journal-remote' to reformat received journals\n"
 188         exit 1
 189     fi
 190     start_backup
 191     incremental
 192     journal
 193 else
 194     printf "Error, unknown sub command: %s\n" "$2" >&2
 195 fi
 196
 197
 198 printf "end\n" >&${COPROC[1]}
 199 read -r line <&${COPROC[0]} || exit 1;
 200 printf "END: %s\n" "$line"