Deploy to Production / Build & Verify (push) Failing after 5m56s
Ping Search Engines / Notify Search Engines (push) Successful in 2s
Deploy to Production / Pre-Deploy Tests (push) Has been skipped
Deploy to Production / Deploy to Railway (push) Has been skipped
Deploy to Production / Deploy to Render (push) Has been skipped
Deploy to Production / Deploy to VPS (PM2) (push) Has been skipped
Deploy to Production / Deploy to Fly.io (push) Has been skipped
Deploy to Production / Post-Deploy Verification (push) Has been skipped
Deploy to Production / Notify on Failure (push) Successful in 2s
E2E Test Suite / Critical User Journeys (push) Has been skipped
E2E Test Suite / API Integration Tests (push) Has been skipped
E2E Test Suite / Smoke Tests (P0) (push) Failing after 11m26s
E2E Test Suite / Form Interaction Tests (push) Failing after 11m42s
E2E Test Suite / Destructive & Chaos Tests (push) Failing after 12m2s
E2E Test Suite / Cross-Browser Regression (chromium) (push) Failing after 16m14s
E2E Test Suite / Cross-Browser Regression (webkit) (push) Failing after 17m45s
E2E Test Suite / Cross-Browser Regression (firefox) (push) Failing after 25m23s
E2E Test Suite / Security Header Tests (push) Failing after 7m55s
E2E Test Suite / Test Report Summary (push) Failing after 20s
E2E Test Suite / Mobile Device Tests (push) Failing after 2h49m9s
Uptime Monitor / Health & Response Time (push) Failing after 2s
Uptime Monitor / SSL Certificate (push) Successful in 2s
Uptime Monitor / Send Alerts (push) Failing after 3s
Uptime Monitor / Record Uptime Success (push) Has been skipped
178 lines
4.8 KiB
Bash
178 lines
4.8 KiB
Bash
#!/bin/bash
|
|
# Backup Verification Script
|
|
# Checks backup health, integrity, and provides status report
|
|
|
|
set -e
|
|
|
|
# Configuration
|
|
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
BACKUP_ROOT="$PROJECT_ROOT/backups"
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m'
|
|
|
|
# Counters
|
|
TOTAL_BACKUPS=0
|
|
VALID_BACKUPS=0
|
|
CORRUPTED_BACKUPS=0
|
|
WARNINGS=0
|
|
|
|
echo -e "${BLUE}=== Backup Verification Report ===${NC}\n"
|
|
echo "Generated: $(date)"
|
|
echo "Backup location: $BACKUP_ROOT"
|
|
echo ""
|
|
|
|
# Function to verify a single backup
|
|
verify_single_backup() {
|
|
local backup_file="$1"
|
|
TOTAL_BACKUPS=$((TOTAL_BACKUPS + 1))
|
|
|
|
if tar -tzf "$backup_file" > /dev/null 2>&1; then
|
|
VALID_BACKUPS=$((VALID_BACKUPS + 1))
|
|
return 0
|
|
else
|
|
CORRUPTED_BACKUPS=$((CORRUPTED_BACKUPS + 1))
|
|
echo -e " ${RED}✗ CORRUPTED: $backup_file${NC}"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# Check daily backups
|
|
echo -e "${YELLOW}Daily Backups:${NC}"
|
|
if [ -d "$BACKUP_ROOT/daily" ]; then
|
|
DAILY_COUNT=$(ls "$BACKUP_ROOT/daily"/*.tar.gz 2>/dev/null | wc -l)
|
|
echo " Found: $DAILY_COUNT backups"
|
|
|
|
if [ "$DAILY_COUNT" -eq 0 ]; then
|
|
echo -e " ${RED}⚠ WARNING: No daily backups found${NC}"
|
|
WARNINGS=$((WARNINGS + 1))
|
|
else
|
|
# Check latest backup age
|
|
LATEST=$(ls -t "$BACKUP_ROOT/daily"/*.tar.gz 2>/dev/null | head -n 1)
|
|
AGE_HOURS=$(( ($(date +%s) - $(stat -c %Y "$LATEST" 2>/dev/null || stat -f %m "$LATEST")) / 3600 ))
|
|
echo " Latest backup: $(basename "$LATEST")"
|
|
echo " Age: ${AGE_HOURS} hours ago"
|
|
|
|
if [ "$AGE_HOURS" -gt 24 ]; then
|
|
echo -e " ${RED}⚠ WARNING: Latest backup is >24 hours old${NC}"
|
|
WARNINGS=$((WARNINGS + 1))
|
|
fi
|
|
|
|
# Verify integrity
|
|
for backup in "$BACKUP_ROOT/daily"/*.tar.gz; do
|
|
verify_single_backup "$backup" > /dev/null
|
|
done
|
|
echo -e " ${GREEN}✓ All daily backups verified${NC}"
|
|
fi
|
|
else
|
|
echo -e " ${RED}✗ Daily backup directory not found${NC}"
|
|
WARNINGS=$((WARNINGS + 1))
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# Check weekly backups
|
|
echo -e "${YELLOW}Weekly Backups:${NC}"
|
|
if [ -d "$BACKUP_ROOT/weekly" ]; then
|
|
WEEKLY_COUNT=$(ls "$BACKUP_ROOT/weekly"/*.tar.gz 2>/dev/null | wc -l)
|
|
echo " Found: $WEEKLY_COUNT backups"
|
|
|
|
if ls "$BACKUP_ROOT/weekly"/*.tar.gz 1> /dev/null 2>&1; then
|
|
for backup in "$BACKUP_ROOT/weekly"/*.tar.gz; do
|
|
verify_single_backup "$backup" > /dev/null
|
|
done
|
|
fi
|
|
|
|
if [ "$WEEKLY_COUNT" -gt 0 ]; then
|
|
echo -e " ${GREEN}✓ All weekly backups verified${NC}"
|
|
fi
|
|
else
|
|
echo " No weekly backups directory"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# Check monthly backups
|
|
echo -e "${YELLOW}Monthly Backups:${NC}"
|
|
if [ -d "$BACKUP_ROOT/monthly" ]; then
|
|
MONTHLY_COUNT=$(ls "$BACKUP_ROOT/monthly"/*.tar.gz 2>/dev/null | wc -l)
|
|
echo " Found: $MONTHLY_COUNT backups"
|
|
|
|
if ls "$BACKUP_ROOT/monthly"/*.tar.gz 1> /dev/null 2>&1; then
|
|
for backup in "$BACKUP_ROOT/monthly"/*.tar.gz; do
|
|
verify_single_backup "$backup" > /dev/null
|
|
done
|
|
fi
|
|
|
|
if [ "$MONTHLY_COUNT" -gt 0 ]; then
|
|
echo -e " ${GREEN}✓ All monthly backups verified${NC}"
|
|
fi
|
|
else
|
|
echo " No monthly backups directory"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# Storage usage
|
|
echo -e "${YELLOW}Storage Usage:${NC}"
|
|
if [ -d "$BACKUP_ROOT" ]; then
|
|
TOTAL_SIZE=$(du -sh "$BACKUP_ROOT" 2>/dev/null | cut -f1)
|
|
echo " Total backup size: $TOTAL_SIZE"
|
|
|
|
# Check if approaching limits (>5GB)
|
|
SIZE_MB=$(du -sm "$BACKUP_ROOT" 2>/dev/null | cut -f1)
|
|
if [ "$SIZE_MB" -gt 5120 ]; then
|
|
echo -e " ${YELLOW}⚠ WARNING: Backup storage >5GB, consider cleanup${NC}"
|
|
WARNINGS=$((WARNINGS + 1))
|
|
fi
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# Check disk space
|
|
echo -e "${YELLOW}Disk Space:${NC}"
|
|
DISK_USAGE=$(df -h "$BACKUP_ROOT" | tail -1)
|
|
DISK_PERCENT=$(echo "$DISK_USAGE" | awk '{print $5}' | sed 's/%//')
|
|
echo " Disk usage: $DISK_PERCENT%"
|
|
|
|
if [ "$DISK_PERCENT" -gt 80 ]; then
|
|
echo -e " ${RED}⚠ WARNING: Disk usage >80%${NC}"
|
|
WARNINGS=$((WARNINGS + 1))
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# Summary
|
|
echo -e "${BLUE}=== Summary ===${NC}"
|
|
echo "Total backups checked: $TOTAL_BACKUPS"
|
|
echo -e "Valid backups: ${GREEN}$VALID_BACKUPS${NC}"
|
|
|
|
if [ "$CORRUPTED_BACKUPS" -gt 0 ]; then
|
|
echo -e "Corrupted backups: ${RED}$CORRUPTED_BACKUPS${NC}"
|
|
fi
|
|
|
|
if [ "$WARNINGS" -gt 0 ]; then
|
|
echo -e "Warnings: ${YELLOW}$WARNINGS${NC}"
|
|
fi
|
|
|
|
echo ""
|
|
|
|
# Final status
|
|
if [ "$CORRUPTED_BACKUPS" -eq 0 ] && [ "$WARNINGS" -eq 0 ] && [ "$VALID_BACKUPS" -gt 0 ]; then
|
|
echo -e "${GREEN}✓ Backup system healthy${NC}"
|
|
exit 0
|
|
elif [ "$CORRUPTED_BACKUPS" -gt 0 ]; then
|
|
echo -e "${RED}✗ CRITICAL: Corrupted backups detected${NC}"
|
|
exit 2
|
|
elif [ "$WARNINGS" -gt 0 ]; then
|
|
echo -e "${YELLOW}⚠ Backup system has warnings${NC}"
|
|
exit 1
|
|
else
|
|
echo -e "${YELLOW}⚠ No backups found - run initial backup${NC}"
|
|
exit 1
|
|
fi
|