mirror of
https://github.com/andsyrovatko/s4k-pve-rsync-backup.git
synced 2026-04-21 13:58:54 +02:00
syr4ok
edc8d166be
- Added PVE vzdump log parsing with awk - Implemented rsync transfer with filelist generation - Added safety triggers for remote cleanup - Added dependency and SSH key validation"
323 lines
13 KiB
Bash
Executable File
323 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# Script Name : pve-rsync-backup.sh
|
|
# Description : Backup Proxmox VE (PVE) virtual machine (VM) images using rsync.
|
|
# The script creates a snapshot of the specified VM, generates a file list of the relevant files,
|
|
# and then uses rsync to copy the files to a backup location. It also handles logging and cleanup of old backups.
|
|
# Usage : ./pve-rsync-backup <Working_node (IP/DOMAIN)> <IMAGE_ID_to_be_backuped (etc. 101)>
|
|
# Author : syr4ok (Andrii Syrovatko)
|
|
# Version : 1.1.4
|
|
# =============================================================================
|
|
|
|
#export LANG=C
|
|
|
|
# Stop script on pipeline errors
|
|
set -o pipefail
|
|
|
|
# Ensure that the script is run with root privileges
|
|
PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
|
|
# Load configuration and set variables
|
|
DATE_NOW=$(date +%Y%m%d)
|
|
TIME_NOW=$(date +%H%M)
|
|
PID=$$
|
|
CONFIG_FILE="$(dirname "$0")/pve-rsync-backup.conf"
|
|
if [[ -f "$CONFIG_FILE" ]]; then
|
|
# shellcheck source=/dev/null
|
|
source "$CONFIG_FILE"
|
|
else
|
|
echo "Error: Configuration file not found. Create pve-rsync-backup.conf from example."
|
|
exit 1
|
|
fi
|
|
|
|
# Exit immediately if a command exits with a non-zero status
|
|
set -e
|
|
|
|
# Formating log-file header
|
|
echo "-----------------$(date '+%H:%M:%S--%Y/%m/%d')-----------------" >> "$TMP_LOGFILE"
|
|
|
|
# CHECK MOUNT BACKUPS PARTITION
|
|
no_rw () {
|
|
log "ERROR" "Disk error mount! Partition ($BASE_DIR) mounted as RO! Call to admin now!"
|
|
if [ "$MAIL_AVAILABLE" = true ]; then
|
|
echo -e "Disk error mount!\nPartition ($BASE_DIR) mounted as RO!\nCall to admin now!" | mail -s "[$SERVER_NAME] WARNING! Backup mnt error(ro mount)" "$SUPPORT_EMAIL"
|
|
else
|
|
log "ERROR" "Mail utility not available, cannot send email alert. Please install 'bsd-mailx' or 'mailutils' to enable email notifications."
|
|
fi
|
|
exit 1
|
|
}
|
|
|
|
# Logging function that outputs to console, backup log file, and system log
|
|
log() {
|
|
local level="$1"
|
|
local message="$2"
|
|
local logfile="$TMP_LOGFILE"
|
|
|
|
# 1. Output to the console (so you can see the process in the terminal)
|
|
local timestamp
|
|
timestamp=$(date '+%H:%M:%S')
|
|
|
|
# 2. Formatted message for console and file
|
|
echo -e "[$timestamp] $level: $message"
|
|
|
|
# 3. Write to the main log file (if a path is specified)
|
|
if [ -n "$logfile" ]; then
|
|
echo -e "$timestamp $level: $message" >> "$logfile"
|
|
fi
|
|
|
|
# 4. Write to the backup local log file (if a path is specified)
|
|
if [ -n "$backup_log" ]; then
|
|
echo -e "$timestamp $level: $message" >> "$backup_log"
|
|
fi
|
|
# 5. System log
|
|
logger -t "${SYSLOG_TAG}[$PID]: $level" "$message"
|
|
}
|
|
|
|
# DEPENDENCY & AUTH CHECK
|
|
log "INFO" "Checking local dependencies and SSH keys..."
|
|
# Check base utilities and commands existence
|
|
for cmd in rsync ssh awk grep; do
|
|
if ! command -v "$cmd" &> /dev/null; then
|
|
log "ERROR" "Dependency missing: '$cmd' is not installed. Install it and try again."
|
|
exit 1
|
|
fi
|
|
done
|
|
# Check if the SSH key file exists and is readable
|
|
if [[ ! -f "$BACKUP_SSH_USER_KEY" ]]; then
|
|
log "ERROR" "SSH Key not found at: $BACKUP_SSH_USER_KEY. Check your config file."
|
|
exit 1
|
|
fi
|
|
# Check SSH key permissions (should be 600 or 400)
|
|
key_perms=$(stat -c "%a" "$BACKUP_SSH_USER_KEY")
|
|
if [[ "$key_perms" != "600" && "$key_perms" != "400" ]]; then
|
|
log "WARN" "SSH Key permissions are too open ($key_perms). Recommended: 600."
|
|
log "WARN" "Attempting to set permissions to 600 for $BACKUP_SSH_USER_KEY..."
|
|
if chmod 600 "$BACKUP_SSH_USER_KEY"; then
|
|
log "INFO" "Permissions for $BACKUP_SSH_USER_KEY successfully set to 600."
|
|
else
|
|
log "ERROR" "Failed to set permissions for $BACKUP_SSH_USER_KEY! Please check manually."
|
|
exit 1
|
|
fi
|
|
fi
|
|
log "INFO" "Dependencies and keys verified."
|
|
|
|
# Check if mail utility is installed
|
|
if ! command -v mail &> /dev/null; then
|
|
log "ERROR" "The 'mail' command is not installed. Please install 'bsd-mailx' or 'mailutils'."
|
|
MAIL_AVAILABLE=false
|
|
else
|
|
MAIL_AVAILABLE=true
|
|
fi
|
|
|
|
# Validate input parameters
|
|
if [[ -z "$1" || -z "$2" ]]; then
|
|
log "ERROR" "Usage: $0 <Working_node (IP/DOMAIN)> <IMAGE_ID_to_be_bckuped (etc. 101)>"
|
|
exit 1
|
|
fi
|
|
if ! [[ "$2" =~ ^[1-9][0-9]*$ ]]; then
|
|
log "ERROR" "Invalid IMAGE_ID: $2. Must be a natural number (positive integer)."
|
|
exit 2
|
|
fi
|
|
|
|
# Check if the backup directory is properly mounted and writable
|
|
LOCAL_BACKUP_DIR="$BASE_BACKUP_DIR"
|
|
check_mount() {
|
|
local target_dir="$LOCAL_BACKUP_DIR"
|
|
|
|
# 1. Check if the directory exists
|
|
if [[ ! -d "$target_dir" ]]; then
|
|
log "ERROR" "Directory $target_dir does NOT exist!"
|
|
exit 1
|
|
fi
|
|
|
|
# 2. Check if it is writable (RW)
|
|
if [[ ! -w "$target_dir" ]]; then
|
|
log "ERROR" "Directory $target_dir is NOT writable (Read-Only or Permission Denied)."
|
|
no_rw
|
|
fi
|
|
|
|
# 3. Informational logging of the status (not mandatory for exit)
|
|
if mountpoint -q "$target_dir"; then
|
|
log "INFO" "$target_dir is a mounted partition. Good."
|
|
else
|
|
log "WARN" "$target_dir is just a directory on the parent partition. Proceeding anyway."
|
|
fi
|
|
}
|
|
# Call 'check_mount' for verification
|
|
check_mount
|
|
|
|
# Assign input parameters to variables for better readability
|
|
server=$1
|
|
vmid=$2
|
|
|
|
# Get the nodename (this is the recognition step)
|
|
# Use substitution to ensure that an empty result does not cause the script to fail via set -e
|
|
log "INFO" "Detecting node name for host $server..."
|
|
NODE_NAME=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
|
-o ConnectTimeout=5 -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" "hostname" 2>/dev/null || echo "FAILED")
|
|
if [[ "$NODE_NAME" == "FAILED" || -z "$NODE_NAME" ]]; then
|
|
log "ERROR" "Could not connect to $server via SSH. Check connectivity/keys."
|
|
exit 2
|
|
fi
|
|
log "INFO" "Target server identified itself as: $NODE_NAME"
|
|
|
|
# NOW check if this node is whitelisted
|
|
node_found=false
|
|
for allowed in "${ALLOWED_NODES[@]}"; do
|
|
[[ "$NODE_NAME" == "$allowed" ]] && node_found=true && break
|
|
done
|
|
if [[ "$node_found" == false ]]; then
|
|
log "ERROR" "Node '$NODE_NAME' (at $server) is not in the ALLOWED_NODES list!"
|
|
exit 2
|
|
fi
|
|
|
|
# If all is good — form the paths
|
|
VM_CFG_PATH="${PVE_CONF_DIR}/${NODE_NAME}/qemu-server/"
|
|
|
|
# Getting needed VM status via SSH
|
|
log "INFO" "Checking status of VM $vmid on $server..."
|
|
vm_status=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
|
-o ConnectTimeout=5 -q -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" "sudo qm status $vmid" 2>&1) || ssh_error=true
|
|
|
|
if [[ "$ssh_error" = true ]]; then
|
|
log "ERROR" "VM $vmid check failed on $server.\nRemote message: $vm_status"
|
|
exit 1
|
|
fi
|
|
log "INFO" "VM Status check passed: $vm_status"
|
|
|
|
# Start the backup process and form the needed directories and files
|
|
log "INFO" "Backup host '$2' at PVE '$1' STARTED:"
|
|
BASE_DIR=/backups/${NODE_NAME}
|
|
if [ ! -d "$BASE_DIR" ]; then
|
|
log "INFO" "BASE dir '$BASE_DIR' not found! Creating..."
|
|
mkdir -p "$BASE_DIR"
|
|
fi
|
|
MIRROR_DIR=${BASE_DIR}/vm${vmid}
|
|
if [ ! -d "$MIRROR_DIR" ]; then
|
|
log "INFO" "MIRROR dir '$MIRROR_DIR' not found! Creating..."
|
|
mkdir -p "$MIRROR_DIR"
|
|
fi
|
|
LOGDIR=${MIRROR_DIR}/logs
|
|
if [ ! -d "$LOGDIR" ]; then
|
|
log "INFO" "LOGS dir '$LOGDIR' not found! Creating..."
|
|
mkdir -p "$LOGDIR"
|
|
fi
|
|
log "INFO" "Creating DATE dir '$MIRROR_DIR/$DATE_NOW'! Creating..."
|
|
backup_dir=${MIRROR_DIR}/${DATE_NOW}_${TIME_NOW}
|
|
mkdir -p "$backup_dir"
|
|
backup_log=${LOGDIR}/${DATE_NOW}_${TIME_NOW}-backup.log
|
|
file_list=${backup_dir}/${vmid}-filelist.txt
|
|
|
|
# Create a snapshot of the VM on the remote PVE server using SSH and vzdump
|
|
echo -e "-----BACKUP PORCESS for '$vmid' at '$server' STARTED-----" > "$backup_log"
|
|
log "INFO" "Creating '$vmid' snapshot at '$server' (remote dir: $PVE_DUMP_DIR). Creating remote snapshot...";
|
|
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" \
|
|
"sudo vzdump $vmid --mode snapshot --compress zstd --dumpdir $PVE_DUMP_DIR/" 2>&1 | tee -a "$backup_log"
|
|
|
|
# Extract the remote dump path from the backup log
|
|
REMOTE_DUMP_PATH=$(grep "creating vzdump archive" "$backup_log" | awk -F"'" '{print $2}' | tr -d '\r\n[:cntrl:]')
|
|
if [[ -n "$REMOTE_DUMP_PATH" ]]; then
|
|
# Form the file list for rsync by finding the relevant files on the remote PVE server using SSH and find
|
|
log "INFO" "Forming filelist '$file_list' for '$vmid'. Forming..."
|
|
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" "sudo find $VM_CFG_PATH -type f -name '*$vmid*'" > "$file_list"
|
|
# ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" "sudo find $PVE_DUMP_DIR -type f -name '*$vmid*' -newermt \"$(date +%Y-%m-%d)\" ! -newermt \"$(date -d 'tomorrow' +%Y-%m-%d)\"" >> "$file_list"
|
|
echo "$REMOTE_DUMP_PATH" >> "$file_list"
|
|
echo "${REMOTE_DUMP_PATH%.vma.zst}.log" >> "$file_list"
|
|
log "INFO" "Rsync filelist for '$vmid' formed with the following content:\n$(cat "$file_list")\n"
|
|
else
|
|
log "ERROR" "Dump path not found! Rsync will have nothing to copy!\n"
|
|
exit 1
|
|
fi
|
|
|
|
# Create a README file in the backup directory with instructions
|
|
log "INFO" "CREATING README.md in $backup_dir..."
|
|
echo -e "VMs config-files location at PVE is (for example):
|
|
/etc/pve/nodes/<node_name>/qemu-server/109.conf
|
|
|
|
COPY ONLY WORKING files from working node!!!
|
|
|
|
Dump location at PVE is (for example):
|
|
/storage/dump/vzdump-qemu-101*.vma.zst
|
|
For restore use on the PVE:
|
|
# qmrestore /storage2/dump/vzdump-qemu-101-2025_04_09-12_07_40.vma.zst 101
|
|
where 101 - id of NEW VM
|
|
"> "${backup_dir}"/Local_README.md
|
|
|
|
# Start the rsync process to copy the files from the remote PVE server to the local backup directory
|
|
log "INFO" "Start RSYNC process for '$vmid' with $file_list. RSYNC starting..."
|
|
rsync_start() {
|
|
/usr/bin/rsync --archive --links --whole-file --numeric-ids \
|
|
--group --owner --perms --times --delete --stats --backup \
|
|
--password-file="$RSYNC_PASS_FILE" \
|
|
--files-from="$file_list" \
|
|
"rsync://${BACKUP_RSYNC_USER}@${server}/${BACKUP_RSYNC_MODULE}" "$backup_dir" \
|
|
>> "$backup_log"
|
|
}
|
|
rsync_start
|
|
rsync_code=$?
|
|
# rsync_code=5 # DEBUG
|
|
|
|
|
|
log "INFO" "rsync's return code: $rsync_code"
|
|
log "INFO" "Details at: ${backup_log}\n"
|
|
|
|
# Check the rsync return code.
|
|
if [ "$rsync_code" -ne 0 ] && [ "$rsync_code" -ne 24 ]; then
|
|
log "ERROR" "Backup failed. Cleaning up local directory..."
|
|
rm -rf "$backup_dir"
|
|
end_log="-----BACKUP PORCESS for '$vmid' at '$server' FAILED-----"
|
|
STATUS_MESSAGE="ERROR: Backup failed with rsync code $rsync_code. Local backup directory cleaned up.\n"
|
|
exit_status=1
|
|
else
|
|
log "INFO" "Backup completed successfully!\n"
|
|
end_log="-----BACKUP PORCESS for '$vmid' at '$server' FINISHED-----"
|
|
STATUS_MESSAGE="INFO: Backup completed successfully!"
|
|
exit_status=0
|
|
fi
|
|
|
|
# Cleanup: Remove the snapshot from the remote PVE server using SSH and find
|
|
log "INFO" "Cleanup: Removing '$vmid' snapshot at remote PVE-server '$server' (dir: $PVE_DUMP_DIR/)..."
|
|
if [[ -n "$REMOTE_DUMP_PATH" && "$REMOTE_DUMP_PATH" == *.vma.zst ]]; then
|
|
REMOTE_LOG_PATH="${REMOTE_DUMP_PATH%.vma.zst}.log"
|
|
log "INFO" "Target dump: $REMOTE_DUMP_PATH"
|
|
log "INFO" "Target log: $REMOTE_LOG_PATH"
|
|
ssh -o StrictHostKeyChecking=no -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER@$server" \
|
|
"sudo rm -v \"$REMOTE_DUMP_PATH\" \"$REMOTE_LOG_PATH\"" >> "$backup_log" 2>&1
|
|
log "INFO" "Remote cleanup finished.\n"
|
|
else
|
|
log "ERROR" "SAFETY TRIGGERED: Will not delete. Path '$REMOTE_DUMP_PATH' is empty or not a .vma.zst file!\n"
|
|
fi
|
|
|
|
# Remove local dumps and log files older than specified days
|
|
if [ "$exit_status" -eq 0 ]; then
|
|
# Cleanup of old backups (dump files) older than specified days:
|
|
log "INFO" "Rotation: Cleaning up backups older than $LOG_RETENTION_DAYS days in $MIRROR_DIR..."
|
|
deleted_dirs=$(find "$MIRROR_DIR" -mindepth 1 -maxdepth 1 -type d -name "20*" -mtime +"$LOG_RETENTION_DAYS" -exec rm -rfv {} \;)
|
|
if [ -n "$deleted_dirs" ]; then
|
|
log "INFO" "Deleted old backup directories:\n${deleted_dirs}\n"
|
|
else
|
|
log "INFO" "No old backups found for deletion.\n"
|
|
fi
|
|
# Cleanup of old logs:
|
|
log "INFO" "Rotation: Cleaning up old logs in $LOGDIR..."
|
|
deleted_logs=$(find "$LOGDIR" -maxdepth 1 -type f -name "*.log" -mtime +"$LOG_RETENTION_DAYS" -exec rm -fv {} \;)
|
|
if [ -n "$deleted_logs" ]; then
|
|
log "INFO" "Deleted old log files:\n${deleted_logs}\n"
|
|
else
|
|
log "INFO" "No old logs found for deletion.\n"
|
|
fi
|
|
else
|
|
log "WARN" "Backup failed, skipping local cleanup of old dumps and logs to preserve data for troubleshooting."
|
|
fi
|
|
|
|
# Send the backup log via email to the support team with the status message and full log details
|
|
echo -e "${end_log}\n\n" >> "$backup_log"
|
|
if [ "$MAIL_AVAILABLE" = true ]; then
|
|
log "INFO" "Sending backup log via email to $SUPPORT_EMAIL..."
|
|
cat "$backup_log" | mail -s "[$SERVER_NAME] ${STATUS_MESSAGE}" "$SUPPORT_EMAIL"
|
|
else
|
|
log "ERROR" "Mail utility not available, cannot send email notification. Please install 'bsd-mailx' or 'mailutils' to enable email notifications."
|
|
fi
|
|
|
|
exit $exit_status
|