mirror of
https://github.com/andsyrovatko/s4k-pve-rsync-backup.git
synced 2026-04-21 22:08:53 +02:00
feat: initial release of pve-rsync-backup script
- Added PVE vzdump log parsing with awk - Implemented rsync transfer with filelist generation - Added safety triggers for remote cleanup - Added dependency and SSH key validation"
This commit is contained in:
Executable
+322
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# Script Name : pve-rsync-backup.sh
|
||||
# Description : Backup Proxmox VE (PVE) virtual machine (VM) images using rsync.
|
||||
# The script creates a snapshot of the specified VM, generates a file list of the relevant files,
|
||||
# and then uses rsync to copy the files to a backup location. It also handles logging and cleanup of old backups.
|
||||
# Usage : ./pve-rsync-backup <Working_node (IP/DOMAIN)> <IMAGE_ID_to_be_backuped (etc. 101)>
|
||||
# Author : syr4ok (Andrii Syrovatko)
|
||||
# Version : 1.1.4
|
||||
# =============================================================================
|
||||
|
||||
#export LANG=C
|
||||
|
||||
# Stop script on pipeline errors
|
||||
set -o pipefail
|
||||
|
||||
# Ensure that the script is run with root privileges
|
||||
PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
||||
|
||||
# Load configuration and set variables
|
||||
DATE_NOW=$(date +%Y%m%d)
|
||||
TIME_NOW=$(date +%H%M)
|
||||
PID=$$
|
||||
CONFIG_FILE="$(dirname "$0")/pve-rsync-backup.conf"
|
||||
if [[ -f "$CONFIG_FILE" ]]; then
|
||||
# shellcheck source=/dev/null
|
||||
source "$CONFIG_FILE"
|
||||
else
|
||||
echo "Error: Configuration file not found. Create pve-rsync-backup.conf from example."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Exit immediately if a command exits with a non-zero status
|
||||
set -e
|
||||
|
||||
# Formating log-file header
|
||||
echo "-----------------$(date '+%H:%M:%S--%Y/%m/%d')-----------------" >> "$TMP_LOGFILE"
|
||||
|
||||
# CHECK MOUNT BACKUPS PARTITION
|
||||
no_rw () {
|
||||
log "ERROR" "Disk error mount! Partition ($BASE_DIR) mounted as RO! Call to admin now!"
|
||||
if [ "$MAIL_AVAILABLE" = true ]; then
|
||||
echo -e "Disk error mount!\nPartition ($BASE_DIR) mounted as RO!\nCall to admin now!" | mail -s "[$SERVER_NAME] WARNING! Backup mnt error(ro mount)" "$SUPPORT_EMAIL"
|
||||
else
|
||||
log "ERROR" "Mail utility not available, cannot send email alert. Please install 'bsd-mailx' or 'mailutils' to enable email notifications."
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Logging function that outputs to console, backup log file, and system log
|
||||
log() {
|
||||
local level="$1"
|
||||
local message="$2"
|
||||
local logfile="$TMP_LOGFILE"
|
||||
|
||||
# 1. Output to the console (so you can see the process in the terminal)
|
||||
local timestamp
|
||||
timestamp=$(date '+%H:%M:%S')
|
||||
|
||||
# 2. Formatted message for console and file
|
||||
echo -e "[$timestamp] $level: $message"
|
||||
|
||||
# 3. Write to the main log file (if a path is specified)
|
||||
if [ -n "$logfile" ]; then
|
||||
echo -e "$timestamp $level: $message" >> "$logfile"
|
||||
fi
|
||||
|
||||
# 4. Write to the backup local log file (if a path is specified)
|
||||
if [ -n "$backup_log" ]; then
|
||||
echo -e "$timestamp $level: $message" >> "$backup_log"
|
||||
fi
|
||||
# 5. System log
|
||||
logger -t "${SYSLOG_TAG}[$PID]: $level" "$message"
|
||||
}
|
||||
|
||||
# DEPENDENCY & AUTH CHECK
|
||||
log "INFO" "Checking local dependencies and SSH keys..."
|
||||
# Check base utilities and commands existence
|
||||
for cmd in rsync ssh awk grep; do
|
||||
if ! command -v "$cmd" &> /dev/null; then
|
||||
log "ERROR" "Dependency missing: '$cmd' is not installed. Install it and try again."
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
# Check if the SSH key file exists and is readable
|
||||
if [[ ! -f "$BACKUP_SSH_USER_KEY" ]]; then
|
||||
log "ERROR" "SSH Key not found at: $BACKUP_SSH_USER_KEY. Check your config file."
|
||||
exit 1
|
||||
fi
|
||||
# Check SSH key permissions (should be 600 or 400)
|
||||
key_perms=$(stat -c "%a" "$BACKUP_SSH_USER_KEY")
|
||||
if [[ "$key_perms" != "600" && "$key_perms" != "400" ]]; then
|
||||
log "WARN" "SSH Key permissions are too open ($key_perms). Recommended: 600."
|
||||
log "WARN" "Attempting to set permissions to 600 for $BACKUP_SSH_USER_KEY..."
|
||||
if chmod 600 "$BACKUP_SSH_USER_KEY"; then
|
||||
log "INFO" "Permissions for $BACKUP_SSH_USER_KEY successfully set to 600."
|
||||
else
|
||||
log "ERROR" "Failed to set permissions for $BACKUP_SSH_USER_KEY! Please check manually."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
log "INFO" "Dependencies and keys verified."
|
||||
|
||||
# Check if mail utility is installed
|
||||
if ! command -v mail &> /dev/null; then
|
||||
log "ERROR" "The 'mail' command is not installed. Please install 'bsd-mailx' or 'mailutils'."
|
||||
MAIL_AVAILABLE=false
|
||||
else
|
||||
MAIL_AVAILABLE=true
|
||||
fi
|
||||
|
||||
# Validate input parameters
|
||||
if [[ -z "$1" || -z "$2" ]]; then
|
||||
log "ERROR" "Usage: $0 <Working_node (IP/DOMAIN)> <IMAGE_ID_to_be_bckuped (etc. 101)>"
|
||||
exit 1
|
||||
fi
|
||||
if ! [[ "$2" =~ ^[1-9][0-9]*$ ]]; then
|
||||
log "ERROR" "Invalid IMAGE_ID: $2. Must be a natural number (positive integer)."
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Check if the backup directory is properly mounted and writable
|
||||
LOCAL_BACKUP_DIR="$BASE_BACKUP_DIR"
|
||||
check_mount() {
|
||||
local target_dir="$LOCAL_BACKUP_DIR"
|
||||
|
||||
# 1. Check if the directory exists
|
||||
if [[ ! -d "$target_dir" ]]; then
|
||||
log "ERROR" "Directory $target_dir does NOT exist!"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 2. Check if it is writable (RW)
|
||||
if [[ ! -w "$target_dir" ]]; then
|
||||
log "ERROR" "Directory $target_dir is NOT writable (Read-Only or Permission Denied)."
|
||||
no_rw
|
||||
fi
|
||||
|
||||
# 3. Informational logging of the status (not mandatory for exit)
|
||||
if mountpoint -q "$target_dir"; then
|
||||
log "INFO" "$target_dir is a mounted partition. Good."
|
||||
else
|
||||
log "WARN" "$target_dir is just a directory on the parent partition. Proceeding anyway."
|
||||
fi
|
||||
}
|
||||
# Call 'check_mount' for verification
|
||||
check_mount
|
||||
|
||||
# Assign input parameters to variables for better readability
|
||||
server=$1
|
||||
vmid=$2
|
||||
|
||||
# Get the nodename (this is the recognition step)
|
||||
# Use substitution to ensure that an empty result does not cause the script to fail via set -e
|
||||
log "INFO" "Detecting node name for host $server..."
|
||||
NODE_NAME=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=5 -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" "hostname" 2>/dev/null || echo "FAILED")
|
||||
if [[ "$NODE_NAME" == "FAILED" || -z "$NODE_NAME" ]]; then
|
||||
log "ERROR" "Could not connect to $server via SSH. Check connectivity/keys."
|
||||
exit 2
|
||||
fi
|
||||
log "INFO" "Target server identified itself as: $NODE_NAME"
|
||||
|
||||
# NOW check if this node is whitelisted
|
||||
node_found=false
|
||||
for allowed in "${ALLOWED_NODES[@]}"; do
|
||||
[[ "$NODE_NAME" == "$allowed" ]] && node_found=true && break
|
||||
done
|
||||
if [[ "$node_found" == false ]]; then
|
||||
log "ERROR" "Node '$NODE_NAME' (at $server) is not in the ALLOWED_NODES list!"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# If all is good — form the paths
|
||||
VM_CFG_PATH="${PVE_CONF_DIR}/${NODE_NAME}/qemu-server/"
|
||||
|
||||
# Getting needed VM status via SSH
|
||||
log "INFO" "Checking status of VM $vmid on $server..."
|
||||
vm_status=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
|
||||
-o ConnectTimeout=5 -q -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" "sudo qm status $vmid" 2>&1) || ssh_error=true
|
||||
|
||||
if [[ "$ssh_error" = true ]]; then
|
||||
log "ERROR" "VM $vmid check failed on $server.\nRemote message: $vm_status"
|
||||
exit 1
|
||||
fi
|
||||
log "INFO" "VM Status check passed: $vm_status"
|
||||
|
||||
# Start the backup process and form the needed directories and files
|
||||
log "INFO" "Backup host '$2' at PVE '$1' STARTED:"
|
||||
BASE_DIR=/backups/${NODE_NAME}
|
||||
if [ ! -d "$BASE_DIR" ]; then
|
||||
log "INFO" "BASE dir '$BASE_DIR' not found! Creating..."
|
||||
mkdir -p "$BASE_DIR"
|
||||
fi
|
||||
MIRROR_DIR=${BASE_DIR}/vm${vmid}
|
||||
if [ ! -d "$MIRROR_DIR" ]; then
|
||||
log "INFO" "MIRROR dir '$MIRROR_DIR' not found! Creating..."
|
||||
mkdir -p "$MIRROR_DIR"
|
||||
fi
|
||||
LOGDIR=${MIRROR_DIR}/logs
|
||||
if [ ! -d "$LOGDIR" ]; then
|
||||
log "INFO" "LOGS dir '$LOGDIR' not found! Creating..."
|
||||
mkdir -p "$LOGDIR"
|
||||
fi
|
||||
log "INFO" "Creating DATE dir '$MIRROR_DIR/$DATE_NOW'! Creating..."
|
||||
backup_dir=${MIRROR_DIR}/${DATE_NOW}_${TIME_NOW}
|
||||
mkdir -p "$backup_dir"
|
||||
backup_log=${LOGDIR}/${DATE_NOW}_${TIME_NOW}-backup.log
|
||||
file_list=${backup_dir}/${vmid}-filelist.txt
|
||||
|
||||
# Create a snapshot of the VM on the remote PVE server using SSH and vzdump
|
||||
echo -e "-----BACKUP PORCESS for '$vmid' at '$server' STARTED-----" > "$backup_log"
|
||||
log "INFO" "Creating '$vmid' snapshot at '$server' (remote dir: $PVE_DUMP_DIR). Creating remote snapshot...";
|
||||
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" \
|
||||
"sudo vzdump $vmid --mode snapshot --compress zstd --dumpdir $PVE_DUMP_DIR/" 2>&1 | tee -a "$backup_log"
|
||||
|
||||
# Extract the remote dump path from the backup log
|
||||
REMOTE_DUMP_PATH=$(grep "creating vzdump archive" "$backup_log" | awk -F"'" '{print $2}' | tr -d '\r\n[:cntrl:]')
|
||||
if [[ -n "$REMOTE_DUMP_PATH" ]]; then
|
||||
# Form the file list for rsync by finding the relevant files on the remote PVE server using SSH and find
|
||||
log "INFO" "Forming filelist '$file_list' for '$vmid'. Forming..."
|
||||
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" "sudo find $VM_CFG_PATH -type f -name '*$vmid*'" > "$file_list"
|
||||
# ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" "sudo find $PVE_DUMP_DIR -type f -name '*$vmid*' -newermt \"$(date +%Y-%m-%d)\" ! -newermt \"$(date -d 'tomorrow' +%Y-%m-%d)\"" >> "$file_list"
|
||||
echo "$REMOTE_DUMP_PATH" >> "$file_list"
|
||||
echo "${REMOTE_DUMP_PATH%.vma.zst}.log" >> "$file_list"
|
||||
log "INFO" "Rsync filelist for '$vmid' formed with the following content:\n$(cat "$file_list")\n"
|
||||
else
|
||||
log "ERROR" "Dump path not found! Rsync will have nothing to copy!\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create a README file in the backup directory with instructions
|
||||
log "INFO" "CREATING README.md in $backup_dir..."
|
||||
echo -e "VMs config-files location at PVE is (for example):
|
||||
/etc/pve/nodes/<node_name>/qemu-server/109.conf
|
||||
|
||||
COPY ONLY WORKING files from working node!!!
|
||||
|
||||
Dump location at PVE is (for example):
|
||||
/storage/dump/vzdump-qemu-101*.vma.zst
|
||||
For restore use on the PVE:
|
||||
# qmrestore /storage2/dump/vzdump-qemu-101-2025_04_09-12_07_40.vma.zst 101
|
||||
where 101 - id of NEW VM
|
||||
"> "${backup_dir}"/Local_README.md
|
||||
|
||||
# Start the rsync process to copy the files from the remote PVE server to the local backup directory
|
||||
log "INFO" "Start RSYNC process for '$vmid' with $file_list. RSYNC starting..."
|
||||
rsync_start() {
|
||||
/usr/bin/rsync --archive --links --whole-file --numeric-ids \
|
||||
--group --owner --perms --times --delete --stats --backup \
|
||||
--password-file="$RSYNC_PASS_FILE" \
|
||||
--files-from="$file_list" \
|
||||
"rsync://${BACKUP_RSYNC_USER}@${server}/${BACKUP_RSYNC_MODULE}" "$backup_dir" \
|
||||
>> "$backup_log"
|
||||
}
|
||||
rsync_start
|
||||
rsync_code=$?
|
||||
# rsync_code=5 # DEBUG
|
||||
|
||||
|
||||
log "INFO" "rsync's return code: $rsync_code"
|
||||
log "INFO" "Details at: ${backup_log}\n"
|
||||
|
||||
# Check the rsync return code.
|
||||
if [ "$rsync_code" -ne 0 ] && [ "$rsync_code" -ne 24 ]; then
|
||||
log "ERROR" "Backup failed. Cleaning up local directory..."
|
||||
rm -rf "$backup_dir"
|
||||
end_log="-----BACKUP PORCESS for '$vmid' at '$server' FAILED-----"
|
||||
STATUS_MESSAGE="ERROR: Backup failed with rsync code $rsync_code. Local backup directory cleaned up.\n"
|
||||
exit_status=1
|
||||
else
|
||||
log "INFO" "Backup completed successfully!\n"
|
||||
end_log="-----BACKUP PORCESS for '$vmid' at '$server' FINISHED-----"
|
||||
STATUS_MESSAGE="INFO: Backup completed successfully!"
|
||||
exit_status=0
|
||||
fi
|
||||
|
||||
# Cleanup: Remove the snapshot from the remote PVE server using SSH and find
|
||||
log "INFO" "Cleanup: Removing '$vmid' snapshot at remote PVE-server '$server' (dir: $PVE_DUMP_DIR/)..."
|
||||
if [[ -n "$REMOTE_DUMP_PATH" && "$REMOTE_DUMP_PATH" == *.vma.zst ]]; then
|
||||
REMOTE_LOG_PATH="${REMOTE_DUMP_PATH%.vma.zst}.log"
|
||||
log "INFO" "Target dump: $REMOTE_DUMP_PATH"
|
||||
log "INFO" "Target log: $REMOTE_LOG_PATH"
|
||||
ssh -o StrictHostKeyChecking=no -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER@$server" \
|
||||
"sudo rm -v \"$REMOTE_DUMP_PATH\" \"$REMOTE_LOG_PATH\"" >> "$backup_log" 2>&1
|
||||
log "INFO" "Remote cleanup finished.\n"
|
||||
else
|
||||
log "ERROR" "SAFETY TRIGGERED: Will not delete. Path '$REMOTE_DUMP_PATH' is empty or not a .vma.zst file!\n"
|
||||
fi
|
||||
|
||||
# Remove local dumps and log files older than specified days
|
||||
if [ "$exit_status" -eq 0 ]; then
|
||||
# Cleanup of old backups (dump files) older than specified days:
|
||||
log "INFO" "Rotation: Cleaning up backups older than $LOG_RETENTION_DAYS days in $MIRROR_DIR..."
|
||||
deleted_dirs=$(find "$MIRROR_DIR" -mindepth 1 -maxdepth 1 -type d -name "20*" -mtime +"$LOG_RETENTION_DAYS" -exec rm -rfv {} \;)
|
||||
if [ -n "$deleted_dirs" ]; then
|
||||
log "INFO" "Deleted old backup directories:\n${deleted_dirs}\n"
|
||||
else
|
||||
log "INFO" "No old backups found for deletion.\n"
|
||||
fi
|
||||
# Cleanup of old logs:
|
||||
log "INFO" "Rotation: Cleaning up old logs in $LOGDIR..."
|
||||
deleted_logs=$(find "$LOGDIR" -maxdepth 1 -type f -name "*.log" -mtime +"$LOG_RETENTION_DAYS" -exec rm -fv {} \;)
|
||||
if [ -n "$deleted_logs" ]; then
|
||||
log "INFO" "Deleted old log files:\n${deleted_logs}\n"
|
||||
else
|
||||
log "INFO" "No old logs found for deletion.\n"
|
||||
fi
|
||||
else
|
||||
log "WARN" "Backup failed, skipping local cleanup of old dumps and logs to preserve data for troubleshooting."
|
||||
fi
|
||||
|
||||
# Send the backup log via email to the support team with the status message and full log details
|
||||
echo -e "${end_log}\n\n" >> "$backup_log"
|
||||
if [ "$MAIL_AVAILABLE" = true ]; then
|
||||
log "INFO" "Sending backup log via email to $SUPPORT_EMAIL..."
|
||||
cat "$backup_log" | mail -s "[$SERVER_NAME] ${STATUS_MESSAGE}" "$SUPPORT_EMAIL"
|
||||
else
|
||||
log "ERROR" "Mail utility not available, cannot send email notification. Please install 'bsd-mailx' or 'mailutils' to enable email notifications."
|
||||
fi
|
||||
|
||||
exit $exit_status
|
||||
Reference in New Issue
Block a user