#!/usr/bin/env bash
# =============================================================================
# Script Name : pve-rsync-backup.sh
# Description : Backup Proxmox VE (PVE) virtual machine (VM) images using rsync.
#               The script creates a snapshot of the specified VM, generates a file list of the relevant files,
#               and then uses rsync to copy the files to a backup location. It also handles logging and cleanup of old backups.
# Usage       : ./pve-rsync-backup <Working_node (IP/DOMAIN)> <IMAGE_ID_to_be_backuped (etc. 101)>
# Author      : syr4ok (Andrii Syrovatko)
# Version     : 1.1.4
# =============================================================================

#export LANG=C

# Stop script on pipeline errors
set -o pipefail

# Ensure that the script is run with root privileges
PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"

# Load configuration and set variables
DATE_NOW=$(date +%Y%m%d)
TIME_NOW=$(date +%H%M)
PID=$$
CONFIG_FILE="$(dirname "$0")/pve-rsync-backup.conf"
if [[ -f "$CONFIG_FILE" ]]; then
    # shellcheck source=/dev/null
    source "$CONFIG_FILE"
else
    echo "Error: Configuration file not found. Create pve-rsync-backup.conf from example."
    exit 1
fi

# Exit immediately if a command exits with a non-zero status
set -e

# Formating log-file header
echo "-----------------$(date '+%H:%M:%S--%Y/%m/%d')-----------------" >> "$TMP_LOGFILE"

# CHECK MOUNT BACKUPS PARTITION
no_rw () {
    log "ERROR" "Disk error mount! Partition ($BASE_DIR) mounted as RO! Call to admin now!"
    if [ "$MAIL_AVAILABLE" = true ]; then
        echo -e "Disk error mount!\nPartition ($BASE_DIR) mounted as RO!\nCall to admin now!" | mail -s "[$SERVER_NAME] WARNING! Backup mnt error(ro mount)" "$SUPPORT_EMAIL"
    else
        log "ERROR" "Mail utility not available, cannot send email alert. Please install 'bsd-mailx' or 'mailutils' to enable email notifications."
    fi
	exit 1
}

# Logging function that outputs to console, backup log file, and system log
log() {
    local level="$1"
    local message="$2"
    local logfile="$TMP_LOGFILE"

    # 1. Output to the console (so you can see the process in the terminal)
    local timestamp
    timestamp=$(date '+%H:%M:%S')

    # 2. Formatted message for console and file
    echo -e "[$timestamp] $level: $message"

    # 3. Write to the main log file (if a path is specified)
    if [ -n "$logfile" ]; then
        echo -e "$timestamp $level: $message" >> "$logfile"
    fi

    # 4. Write to the backup local log file (if a path is specified)
    if [ -n "$backup_log" ]; then
        echo -e "$timestamp $level: $message" >> "$backup_log"
    fi
    # 5. System log
    logger -t "${SYSLOG_TAG}[$PID]: $level" "$message"
}

# DEPENDENCY & AUTH CHECK
log "INFO" "Checking local dependencies and SSH keys..."
# Check base utilities and commands existence
for cmd in rsync ssh awk grep; do
    if ! command -v "$cmd" &> /dev/null; then
        log "ERROR" "Dependency missing: '$cmd' is not installed. Install it and try again."
        exit 1
    fi
done
# Check if the SSH key file exists and is readable
if [[ ! -f "$BACKUP_SSH_USER_KEY" ]]; then
    log "ERROR" "SSH Key not found at: $BACKUP_SSH_USER_KEY. Check your config file."
    exit 1
fi
# Check SSH key permissions (should be 600 or 400)
key_perms=$(stat -c "%a" "$BACKUP_SSH_USER_KEY")
if [[ "$key_perms" != "600" && "$key_perms" != "400" ]]; then
    log "WARN" "SSH Key permissions are too open ($key_perms). Recommended: 600."
    log "WARN" "Attempting to set permissions to 600 for $BACKUP_SSH_USER_KEY..."
    if chmod 600 "$BACKUP_SSH_USER_KEY"; then
        log "INFO" "Permissions for $BACKUP_SSH_USER_KEY successfully set to 600."
    else
        log "ERROR" "Failed to set permissions for $BACKUP_SSH_USER_KEY! Please check manually."
        exit 1
    fi
fi
log "INFO" "Dependencies and keys verified."

# Check if mail utility is installed
if ! command -v mail &> /dev/null; then
    log "ERROR" "The 'mail' command is not installed. Please install 'bsd-mailx' or 'mailutils'."
    MAIL_AVAILABLE=false
else
    MAIL_AVAILABLE=true
fi

# Validate input parameters
if [[ -z "$1" || -z "$2" ]]; then
	log "ERROR" "Usage: $0 <Working_node (IP/DOMAIN)> <IMAGE_ID_to_be_bckuped (etc. 101)>"
	exit 1
fi
if ! [[ "$2" =~ ^[1-9][0-9]*$ ]]; then
	log "ERROR" "Invalid IMAGE_ID: $2. Must be a natural number (positive integer)."
	exit 2
fi

# Check if the backup directory is properly mounted and writable
LOCAL_BACKUP_DIR="$BASE_BACKUP_DIR"
check_mount() {
    local target_dir="$LOCAL_BACKUP_DIR"

    # 1. Check if the directory exists
    if [[ ! -d "$target_dir" ]]; then
        log "ERROR" "Directory $target_dir does NOT exist!"
        exit 1
    fi

    # 2. Check if it is writable (RW)
    if [[ ! -w "$target_dir" ]]; then
        log "ERROR" "Directory $target_dir is NOT writable (Read-Only or Permission Denied)."
        no_rw
    fi

    # 3. Informational logging of the status (not mandatory for exit)
    if mountpoint -q "$target_dir"; then
        log "INFO" "$target_dir is a mounted partition. Good."
    else
        log "WARN" "$target_dir is just a directory on the parent partition. Proceeding anyway."
    fi
}
# Call 'check_mount' for verification
check_mount

# Assign input parameters to variables for better readability
server=$1
vmid=$2

# Get the nodename (this is the recognition step)
# Use substitution to ensure that an empty result does not cause the script to fail via set -e
log "INFO" "Detecting node name for host $server..."
NODE_NAME=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
    -o ConnectTimeout=5 -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" "hostname" 2>/dev/null || echo "FAILED")
if [[ "$NODE_NAME" == "FAILED" || -z "$NODE_NAME" ]]; then
    log "ERROR" "Could not connect to $server via SSH. Check connectivity/keys."
    exit 2
fi
log "INFO" "Target server identified itself as: $NODE_NAME"

# NOW check if this node is whitelisted
node_found=false
for allowed in "${ALLOWED_NODES[@]}"; do
    [[ "$NODE_NAME" == "$allowed" ]] && node_found=true && break
done
if [[ "$node_found" == false ]]; then
    log "ERROR" "Node '$NODE_NAME' (at $server) is not in the ALLOWED_NODES list!"
    exit 2
fi

# If all is good — form the paths
VM_CFG_PATH="${PVE_CONF_DIR}/${NODE_NAME}/qemu-server/"

# Getting needed VM status via SSH
log "INFO" "Checking status of VM $vmid on $server..."
vm_status=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \
    -o ConnectTimeout=5 -q -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" "sudo qm status $vmid" 2>&1) || ssh_error=true

if [[ "$ssh_error" = true ]]; then
    log "ERROR" "VM $vmid check failed on $server.\nRemote message: $vm_status"
    exit 1
fi
log "INFO" "VM Status check passed: $vm_status"

# Start the backup process and form the needed directories and files
log "INFO" "Backup host '$2' at PVE '$1' STARTED:"
BASE_DIR=/backups/${NODE_NAME}
if [ ! -d "$BASE_DIR" ]; then
    log "INFO" "BASE dir '$BASE_DIR' not found! Creating..."
    mkdir -p "$BASE_DIR"
fi
MIRROR_DIR=${BASE_DIR}/vm${vmid}
if [ ! -d "$MIRROR_DIR" ]; then
	log "INFO" "MIRROR dir '$MIRROR_DIR' not found! Creating..."
	mkdir -p "$MIRROR_DIR"
fi
LOGDIR=${MIRROR_DIR}/logs
if [ ! -d "$LOGDIR" ]; then
	log "INFO" "LOGS dir '$LOGDIR' not found! Creating..."
	mkdir -p "$LOGDIR"
fi
log "INFO" "Creating DATE dir '$MIRROR_DIR/$DATE_NOW'! Creating..."
backup_dir=${MIRROR_DIR}/${DATE_NOW}_${TIME_NOW}
mkdir -p "$backup_dir"
backup_log=${LOGDIR}/${DATE_NOW}_${TIME_NOW}-backup.log
file_list=${backup_dir}/${vmid}-filelist.txt

# Create a snapshot of the VM on the remote PVE server using SSH and vzdump
echo -e "-----BACKUP PORCESS for '$vmid' at '$server' STARTED-----" > "$backup_log"
log "INFO" "Creating '$vmid' snapshot at '$server' (remote dir: $PVE_DUMP_DIR). Creating remote snapshot...";
ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" \
    "sudo vzdump $vmid --mode snapshot --compress zstd --dumpdir $PVE_DUMP_DIR/" 2>&1 | tee -a "$backup_log"

# Extract the remote dump path from the backup log
REMOTE_DUMP_PATH=$(grep "creating vzdump archive" "$backup_log" | awk -F"'" '{print $2}' | tr -d '\r\n[:cntrl:]')
if [[ -n "$REMOTE_DUMP_PATH" ]]; then
    # Form the file list for rsync by finding the relevant files on the remote PVE server using SSH and find
    log "INFO" "Forming filelist '$file_list' for '$vmid'. Forming..."
    ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" "sudo find $VM_CFG_PATH -type f -name '*$vmid*'" > "$file_list"
    # ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER"@"$server" "sudo find $PVE_DUMP_DIR -type f -name '*$vmid*' -newermt \"$(date +%Y-%m-%d)\" ! -newermt \"$(date -d 'tomorrow' +%Y-%m-%d)\"" >> "$file_list"
    echo "$REMOTE_DUMP_PATH" >> "$file_list"
    echo "${REMOTE_DUMP_PATH%.vma.zst}.log" >> "$file_list"
    log "INFO" "Rsync filelist for '$vmid' formed with the following content:\n$(cat "$file_list")\n"
else
    log "ERROR" "Dump path not found! Rsync will have nothing to copy!\n"
    exit 1
fi

# Create a README file in the backup directory with instructions
log "INFO" "CREATING README.md in $backup_dir..."
echo -e "VMs config-files location at PVE is (for example):
/etc/pve/nodes/<node_name>/qemu-server/109.conf

COPY ONLY WORKING files from working node!!!

Dump location at PVE is (for example):
/storage/dump/vzdump-qemu-101*.vma.zst
For restore use on the PVE:
# qmrestore /storage2/dump/vzdump-qemu-101-2025_04_09-12_07_40.vma.zst 101
where 101 - id of NEW VM
"> "${backup_dir}"/Local_README.md

# Start the rsync process to copy the files from the remote PVE server to the local backup directory
log "INFO" "Start RSYNC process for '$vmid' with $file_list. RSYNC starting..."
rsync_start() {
    /usr/bin/rsync --archive --links --whole-file --numeric-ids \
        --group --owner --perms --times --delete --stats --backup \
        --password-file="$RSYNC_PASS_FILE" \
        --files-from="$file_list" \
        "rsync://${BACKUP_RSYNC_USER}@${server}/${BACKUP_RSYNC_MODULE}" "$backup_dir" \
    >> "$backup_log"
}
rsync_start
rsync_code=$?
# rsync_code=5 # DEBUG


log "INFO" "rsync's return code: $rsync_code"
log "INFO" "Details at: ${backup_log}\n"

# Check the rsync return code.
if [ "$rsync_code" -ne 0 ] && [ "$rsync_code" -ne 24 ]; then
    log "ERROR" "Backup failed. Cleaning up local directory..."
    rm -rf "$backup_dir"
    end_log="-----BACKUP PORCESS for '$vmid' at '$server' FAILED-----"
    STATUS_MESSAGE="ERROR: Backup failed with rsync code $rsync_code. Local backup directory cleaned up.\n"
    exit_status=1
else
    log "INFO" "Backup completed successfully!\n"
    end_log="-----BACKUP PORCESS for '$vmid' at '$server' FINISHED-----"
    STATUS_MESSAGE="INFO: Backup completed successfully!"
    exit_status=0
fi

# Cleanup: Remove the snapshot from the remote PVE server using SSH and find
log "INFO" "Cleanup: Removing '$vmid' snapshot at remote PVE-server '$server' (dir: $PVE_DUMP_DIR/)..."
if [[ -n "$REMOTE_DUMP_PATH" && "$REMOTE_DUMP_PATH" == *.vma.zst ]]; then
    REMOTE_LOG_PATH="${REMOTE_DUMP_PATH%.vma.zst}.log"
    log "INFO" "Target dump: $REMOTE_DUMP_PATH"
    log "INFO" "Target log: $REMOTE_LOG_PATH"
    ssh -o StrictHostKeyChecking=no -i "$BACKUP_SSH_USER_KEY" "$BACKUP_SSH_USER@$server" \
        "sudo rm -v \"$REMOTE_DUMP_PATH\" \"$REMOTE_LOG_PATH\"" >> "$backup_log" 2>&1
    log "INFO" "Remote cleanup finished.\n"
else
    log "ERROR" "SAFETY TRIGGERED: Will not delete. Path '$REMOTE_DUMP_PATH' is empty or not a .vma.zst file!\n"
fi

# Remove local dumps and log files older than specified days
if [ "$exit_status" -eq 0 ]; then
    # Cleanup of old backups (dump files) older than specified days:
    log "INFO" "Rotation: Cleaning up backups older than $LOG_RETENTION_DAYS days in $MIRROR_DIR..."
    deleted_dirs=$(find "$MIRROR_DIR" -mindepth 1 -maxdepth 1 -type d -name "20*" -mtime +"$LOG_RETENTION_DAYS" -exec rm -rfv {} \;)
    if [ -n "$deleted_dirs" ]; then
        log "INFO" "Deleted old backup directories:\n${deleted_dirs}\n"
    else
        log "INFO" "No old backups found for deletion.\n"
    fi
    # Cleanup of old logs:
    log "INFO" "Rotation: Cleaning up old logs in $LOGDIR..."
    deleted_logs=$(find "$LOGDIR" -maxdepth 1 -type f -name "*.log" -mtime +"$LOG_RETENTION_DAYS" -exec rm -fv {} \;)
    if [ -n "$deleted_logs" ]; then
        log "INFO" "Deleted old log files:\n${deleted_logs}\n"
    else
        log "INFO" "No old logs found for deletion.\n"
    fi
else
    log "WARN" "Backup failed, skipping local cleanup of old dumps and logs to preserve data for troubleshooting."
fi

# Send the backup log via email to the support team with the status message and full log details
echo -e "${end_log}\n\n" >> "$backup_log"
if [ "$MAIL_AVAILABLE" = true ]; then
    log "INFO" "Sending backup log via email to $SUPPORT_EMAIL..."
    cat "$backup_log" | mail -s "[$SERVER_NAME] ${STATUS_MESSAGE}" "$SUPPORT_EMAIL"
else
    log "ERROR" "Mail utility not available, cannot send email notification. Please install 'bsd-mailx' or 'mailutils' to enable email notifications."
fi

exit $exit_status