From 6ff30f729479d6a81afa9361088a5cd283c3773a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Nov 2024 10:08:30 +0100 Subject: [PATCH 01/10] Backup a stripped-down version of gravity.db after each pihole -g run Signed-off-by: DL6ER --- gravity.sh | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index a657727d..1c4eb2cb 100755 --- a/gravity.sh +++ b/gravity.sh @@ -58,6 +58,7 @@ gravityDBfile_default="/etc/pihole/gravity.db" gravityTEMPfile="${GRAVITYDB}_temp" gravityDIR="$(dirname -- "${gravityDBfile}")" gravityOLDfile="${gravityDIR}/gravity_old.db" +gravityBCKfile="${gravityDIR}/gravity_backup.db" # Generate new SQLite3 file from schema template generate_gravity_database() { @@ -86,6 +87,15 @@ gravity_build_tree() { echo -e "${OVER} ${TICK} ${str}" } +# Rotate gravity backup files +rotate_gravity_backup() { + for i in {9..1}; do + if [ -f "${gravityBCKfile}.${i}" ]; then + mv "${gravityBCKfile}.${i}" "${gravityBCKfile}.$((i + 1))" + fi + done +} + # Copy data from old to new database file and swap them gravity_swap_databases() { str="Swapping databases" @@ -101,10 +111,28 @@ gravity_swap_databases() { oldAvail=false if [ "${availableBlocks}" -gt "$((gravityBlocks * 2))" ] && [ -f "${gravityDBfile}" ]; then oldAvail=true - mv "${gravityDBfile}" "${gravityOLDfile}" - else - rm "${gravityDBfile}" + cp "${gravityDBfile}" "${gravityOLDfile}" fi + + # Drop the gravity and antigravity tables + subsequent VACUUM the current + # database for compaction + output=$({ printf ".timeout 30000\\nDROP TABLE IF EXISTS gravity;\\nDROP TABLE IF EXISTS antigravity;\\nVACUUM;\\n" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to clean current database for backup\\n ${output}" + rotate=false + else + # If multiple gravityBCKfile's are present (appended with a number), rotate them + # We keep at most 10 backups + rotate_gravity_backup + + # Move the old database to the backup location + mv "${gravityDBfile}" "${gravityBCKfile}.1" + fi + + + # Move the new database to the correct location mv "${gravityTEMPfile}" "${gravityDBfile}" echo -e "${OVER} ${TICK} ${str}" From b2ad878f4ac6742ac018161f9f05077c6dd8cd1d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Nov 2024 10:18:00 +0100 Subject: [PATCH 02/10] Try to restore from gravity backup on database preparation error (if backup is available) Signed-off-by: DL6ER --- gravity.sh | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 1c4eb2cb..7b1b0987 100755 --- a/gravity.sh +++ b/gravity.sh @@ -345,6 +345,43 @@ gravity_CheckDNSResolutionAvailable() { echo -e "${OVER} ${TICK} DNS resolution is available" } +# Function: try_restore_backup +# Description: Attempts to restore the previous Pi-hole gravity database from a +# backup file. If a backup exists, it copies the backup to the +# gravity database file and prepares a new gravity database. If the +# restoration is successful, it returns 0. Otherwise, it returns 1. +# Returns: +# 0 - If the backup is successfully restored. +# 1 - If no backup is available or if the restoration fails. +try_restore_backup () { + # Check if a backup exists + if [ -f "${gravityBCKfile}.1" ]; then + echo -e " ${INFO} Attempting to restore previous database from backup" + cp "${gravityBCKfile}.1" "${gravityDBfile}" + + # If the backup was successfully copied, prepare a new gravity database from + # it + if [ -f "${gravityDBfile}" ]; then + output=$({ pihole-FTL sqlite3 -ni "${gravityTEMPfile}" <<<"${copyGravity}"; } 2>&1) + status="$?" + + # Error checking + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" + gravity_Cleanup "error" + fi + + echo -e " ${TICK} Successfully restored from backup (${gravityBCKfile}.1)" + return 0 + else + echo -e " ${CROSS} Unable to restore backup" + fi + fi + + echo -e " ${CROSS} No backup available" + return 1 +} + # Retrieve blocklist URLs and parse domains from adlist.list gravity_DownloadBlocklists() { echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..." @@ -411,7 +448,11 @@ gravity_DownloadBlocklists() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" - return 1 + + # Try to attempt a backup restore + if ! try_restore_backup; then + return 1 + fi fi echo -e "${OVER} ${TICK} ${str}" From 59e9bac79482e9d1a54ec4b654372a305ef8e079 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Nov 2024 11:47:59 +0100 Subject: [PATCH 03/10] Only try to obtain sources after possible database restore following a corruption Signed-off-by: DL6ER --- gravity.sh | 56 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 27 deletions(-) diff --git a/gravity.sh b/gravity.sh index 7b1b0987..9b8eec11 100755 --- a/gravity.sh +++ b/gravity.sh @@ -121,7 +121,6 @@ gravity_swap_databases() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to clean current database for backup\\n ${output}" - rotate=false else # If multiple gravityBCKfile's are present (appended with a number), rotate them # We keep at most 10 backups @@ -390,32 +389,6 @@ gravity_DownloadBlocklists() { echo -e " ${INFO} Storing gravity database in ${COL_BOLD}${gravityDBfile}${COL_NC}" fi - # Retrieve source URLs from gravity database - # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)" - mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)" - mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)" - - # Parse source domains from $sources - mapfile -t sourceDomains <<<"$( - # Logic: Split by folder/port - awk -F '[/:]' '{ - # Remove URL protocol & optional username:password@ - gsub(/(.*:\/\/|.*:.*@)/, "", $0) - if(length($1)>0){print $1} - else {print "local"} - }' <<<"$(printf '%s\n' "${sources[@]}")" 2>/dev/null - )" - - local str="Pulling blocklist source list into range" - echo -e "${OVER} ${TICK} ${str}" - - if [[ -z "${sources[*]}" ]] || [[ -z "${sourceDomains[*]}" ]]; then - echo -e " ${INFO} No source list found, or it is empty" - echo "" - unset sources - fi - local url domain str target compression adlist_type echo "" @@ -453,9 +426,38 @@ gravity_DownloadBlocklists() { if ! try_restore_backup; then return 1 fi + + echo -e " ${TICK} ${str}" + else + echo -e "${OVER} ${TICK} ${str}" fi + + # Retrieve source URLs from gravity database + # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) + mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)" + mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)" + mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)" + + # Parse source domains from $sources + mapfile -t sourceDomains <<<"$( + # Logic: Split by folder/port + awk -F '[/:]' '{ + # Remove URL protocol & optional username:password@ + gsub(/(.*:\/\/|.*:.*@)/, "", $0) + if(length($1)>0){print $1} + else {print "local"} + }' <<<"$(printf '%s\n' "${sources[@]}")" 2>/dev/null + )" + + local str="Pulling blocklist source list into range" echo -e "${OVER} ${TICK} ${str}" + if [[ -z "${sources[*]}" ]] || [[ -z "${sourceDomains[*]}" ]]; then + echo -e " ${INFO} No source list found, or it is empty" + echo "" + unset sources + fi + # Use compression to reduce the amount of data that is transferred # between the Pi-hole and the ad list provider. Use this feature # only if it is supported by the locally available version of curl From a5cb07c76ef196b3991614892683ec259d2e4b8e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Nov 2024 11:52:31 +0100 Subject: [PATCH 04/10] Attempt to restore from all possibly available gravity.db backups Signed-off-by: DL6ER --- gravity.sh | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 9b8eec11..3ca775a4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -353,10 +353,12 @@ gravity_CheckDNSResolutionAvailable() { # 0 - If the backup is successfully restored. # 1 - If no backup is available or if the restoration fails. try_restore_backup () { + local num + num=$1 # Check if a backup exists - if [ -f "${gravityBCKfile}.1" ]; then + if [ -f "${gravityBCKfile}.${num}" ]; then echo -e " ${INFO} Attempting to restore previous database from backup" - cp "${gravityBCKfile}.1" "${gravityDBfile}" + cp "${gravityBCKfile}.${num}" "${gravityDBfile}" # If the backup was successfully copied, prepare a new gravity database from # it @@ -370,14 +372,14 @@ try_restore_backup () { gravity_Cleanup "error" fi - echo -e " ${TICK} Successfully restored from backup (${gravityBCKfile}.1)" + echo -e " ${TICK} Successfully restored from backup (${gravityBCKfile}.${num})" return 0 else - echo -e " ${CROSS} Unable to restore backup" + echo -e " ${CROSS} Unable to restore backup no. ${num}" fi fi - echo -e " ${CROSS} No backup available" + echo -e " ${CROSS} Backup no. ${num} not available" return 1 } @@ -423,7 +425,14 @@ gravity_DownloadBlocklists() { echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" # Try to attempt a backup restore - if ! try_restore_backup; then + for i in {1..9}; do + if try_restore_backup "${i}"; then + break + fi + done + + # If none of the attempts worked, return 1 + if [[ "${i}" -eq 9 ]]; then return 1 fi From fdf44355d2dc6aad23b269aff29f61f4ff700706 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Nov 2024 12:08:27 +0100 Subject: [PATCH 05/10] Keep up to 10 database backups Signed-off-by: DL6ER --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 3ca775a4..ebef5d2f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -425,14 +425,14 @@ gravity_DownloadBlocklists() { echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" # Try to attempt a backup restore - for i in {1..9}; do + for i in {1..10}; do if try_restore_backup "${i}"; then break fi done # If none of the attempts worked, return 1 - if [[ "${i}" -eq 9 ]]; then + if [[ "${i}" -eq 10 ]]; then return 1 fi From 9a9af719f030ed39c0d87f4b8e3412f95deed446 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sun, 17 Nov 2024 17:05:26 +0100 Subject: [PATCH 06/10] Update gravity.sh Co-authored-by: RD WebDesign Signed-off-by: Dominik --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index ebef5d2f..b616b2e4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -357,7 +357,7 @@ try_restore_backup () { num=$1 # Check if a backup exists if [ -f "${gravityBCKfile}.${num}" ]; then - echo -e " ${INFO} Attempting to restore previous database from backup" + echo -e " ${INFO} Attempting to restore previous database from backup no. ${num}" cp "${gravityBCKfile}.${num}" "${gravityDBfile}" # If the backup was successfully copied, prepare a new gravity database from From 633c971a4433f171acecf38176d849d5c2beac93 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 23 Nov 2024 09:41:59 +0100 Subject: [PATCH 07/10] Store failed gravity restoration status in message table Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 5 +++++ gravity.sh | 9 ++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 5436fb87..3e8603e9 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -67,6 +67,11 @@ CREATE TABLE info ); INSERT INTO "info" VALUES('version','19'); +/* This is a flag to indicate if gravity was restored from a backup + false = not restored, + failed = restoration failed due to no backup + other string = restoration successful with the string being the backup file used */ +INSERT INTO "info" VALUES('gravity_restored','false'); CREATE TABLE domain_audit ( diff --git a/gravity.sh b/gravity.sh index b616b2e4..4bf8e9c6 100755 --- a/gravity.sh +++ b/gravity.sh @@ -353,12 +353,13 @@ gravity_CheckDNSResolutionAvailable() { # 0 - If the backup is successfully restored. # 1 - If no backup is available or if the restoration fails. try_restore_backup () { - local num + local num filename num=$1 + filename="${gravityBCKfile}.${num}" # Check if a backup exists - if [ -f "${gravityBCKfile}.${num}" ]; then + if [ -f "${filename}" ]; then echo -e " ${INFO} Attempting to restore previous database from backup no. ${num}" - cp "${gravityBCKfile}.${num}" "${gravityDBfile}" + cp "${filename}" "${gravityDBfile}" # If the backup was successfully copied, prepare a new gravity database from # it @@ -372,6 +373,7 @@ try_restore_backup () { gravity_Cleanup "error" fi + pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','${filename}');" echo -e " ${TICK} Successfully restored from backup (${gravityBCKfile}.${num})" return 0 else @@ -434,6 +436,7 @@ gravity_DownloadBlocklists() { # If none of the attempts worked, return 1 if [[ "${i}" -eq 10 ]]; then return 1 + pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','failed');" fi echo -e " ${TICK} ${str}" From 12927f8a3d85a70983c04ed07a0f298ae7c94b6c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 12 Jan 2025 13:41:27 +0100 Subject: [PATCH 08/10] Put gravity backup files into dedicated directory for cleaness Signed-off-by: DL6ER --- gravity.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index ee00741a..0fe5690b 100755 --- a/gravity.sh +++ b/gravity.sh @@ -58,7 +58,8 @@ gravityDBfile_default="/etc/pihole/gravity.db" gravityTEMPfile="${GRAVITYDB}_temp" gravityDIR="$(dirname -- "${gravityDBfile}")" gravityOLDfile="${gravityDIR}/gravity_old.db" -gravityBCKfile="${gravityDIR}/gravity_backup.db" +gravityBCKdir="${gravityDIR}/gravity_backups" +gravityBCKfile="${gravityBCKdir}/gravity.db" fix_owner_permissions() { # Fix ownership and permissions for the specified file @@ -132,6 +133,11 @@ gravity_swap_databases() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to clean current database for backup\\n ${output}" else + # Check if the backup directory exists + if [ ! -d "${gravityBCKdir}" ]; then + mkdir -p "${gravityBCKdir}" + fi + # If multiple gravityBCKfile's are present (appended with a number), rotate them # We keep at most 10 backups rotate_gravity_backup @@ -434,11 +440,13 @@ gravity_DownloadBlocklists() { echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" # Try to attempt a backup restore - for i in {1..10}; do - if try_restore_backup "${i}"; then - break - fi - done + if [[ -d "${gravityBCKdir}" ]]; then + for i in {1..10}; do + if try_restore_backup "${i}"; then + break + fi + done + fi # If none of the attempts worked, return 1 if [[ "${i}" -eq 10 ]]; then From cc25ee940e00d211c94cdbad1575c19eb634a35b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 12 Jan 2025 13:42:51 +0100 Subject: [PATCH 09/10] Fix unreachable code Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 0fe5690b..f35eb674 100755 --- a/gravity.sh +++ b/gravity.sh @@ -450,8 +450,8 @@ gravity_DownloadBlocklists() { # If none of the attempts worked, return 1 if [[ "${i}" -eq 10 ]]; then - return 1 pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','failed');" + return 1 fi echo -e " ${TICK} ${str}" From 69bfb3ff3b955d3dfb76e3095e18b037b52eb0d6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 26 Jan 2025 19:52:46 +0100 Subject: [PATCH 10/10] Store timestamp of the backup creation instead of the filename on restoring gravity.db Signed-off-by: DL6ER --- gravity.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index f35eb674..68840b51 100755 --- a/gravity.sh +++ b/gravity.sh @@ -366,7 +366,7 @@ gravity_CheckDNSResolutionAvailable() { # 0 - If the backup is successfully restored. # 1 - If no backup is available or if the restoration fails. try_restore_backup () { - local num filename + local num filename timestamp num=$1 filename="${gravityBCKfile}.${num}" # Check if a backup exists @@ -386,8 +386,15 @@ try_restore_backup () { gravity_Cleanup "error" fi - pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','${filename}');" - echo -e " ${TICK} Successfully restored from backup (${gravityBCKfile}.${num})" + # Get the timestamp of the backup file in a human-readable format + # Note that this timestamp will be in the server timezone, this may be + # GMT, e.g., on a Raspberry Pi where the default timezone has never been + # changed + timestamp=$(date -r "${filename}" "+%Y-%m-%d %H:%M:%S %Z") + + # Add a record to the info table to indicate that the gravity database was restored + pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) values ('gravity_restored','${timestamp}');" + echo -e " ${TICK} Successfully restored from backup (${gravityBCKfile}.${num} at ${timestamp})" return 0 else echo -e " ${CROSS} Unable to restore backup no. ${num}"