From 190ab796063a53ac9843fd97cc8f978d10bd4a56 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 20 Aug 2021 20:48:57 +0200 Subject: [PATCH 1/7] Implement fully-automated gravity database recovery method. Signed-off-by: DL6ER --- gravity.sh | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 62 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index b2cd27b9..911242f7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -848,6 +848,42 @@ gravity_Cleanup() { fi } +database_recovery() { + str="Checking integrity of existing gravity database" + echo -ne " ${INFO} ${str}..." + if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)"; then + echo -e "${OVER} ${TICK} ${str} - no errors found" + else + echo -e "${OVER} ${CROSS} ${str} - errors found:" + while IFS= read -r line ; do echo " - $line"; done <<< "$result" + fi + + str="Checking foreign keys of existing gravity database" + echo -ne " ${INFO} ${str}..." + if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"; then + echo -e "${OVER} ${TICK} ${str} - no errors found" + else + echo -e "${OVER} ${CROSS} ${str} - errors found:" + while IFS= read -r line ; do echo " - $line"; done <<< "$result" + fi + + str="Trying to recover existing gravity database" + echo -ne " ${INFO} ${str}..." + # We have to remove any possibly existing recovery database or this will fail + rm -f "${gravityDBfile}.recovered" > /dev/null 2>&1 + if result="$(pihole-FTL sqlite3 "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 "${gravityDBfile}.recovered" 2>&1)"; then + echo -e "${OVER} ${TICK} ${str} - success" + mv "${gravityDBfile}" "${gravityDBfile}.old" + mv "${gravityDBfile}.recovered" "${gravityDBfile}" + else + echo -e "${OVER} ${CROSS} ${str} - the following errors happened:" + while IFS= read -r line ; do echo " - $line"; done <<< "$result" + echo -e " ${CROSS} Recovery failed. Try \"pihole -r recreate\" instead." + exit 1 + fi + echo "" +} + helpFunc() { echo "Usage: pihole -g Update domains from blocklists specified in adlists.list @@ -858,10 +894,30 @@ Options: exit 0 } +repairSelector() { + case "$1" in + "recover") recover_database=true;; + "recreate") recreate_database=true;; + *) echo "Usage: pihole -g -r {recover,recreate} +Attempt to repair gravity database + +Available options: + pihole -g -r recover Try to recover a damaged gravity database file. + Pi-hole tries to restore as much as possible + from a corrupted gravity database. + pihole -g -r recreate Create a new gravity database file from scratch. + This will remove your existing gravity database + and create a new file from scratch. If you still + have the migration backup created when migrating + to Pi-hole v5.0, Pi-hole will import these files." + exit 0;; + esac +} + for var in "$@"; do case "${var}" in "-f" | "--force" ) forceDelete=true;; - "-r" | "--recreate" ) recreate_database=true;; + "-r" | "--repair" ) repairSelector "$3";; "-h" | "--help" ) helpFunc;; esac done @@ -875,7 +931,7 @@ fi gravity_Trap if [[ "${recreate_database:-}" == true ]]; then - str="Restoring from migration backup" + str="Recreating gravity database from migration backup" echo -ne "${INFO} ${str}..." rm "${gravityDBfile}" pushd "${piholeDir}" > /dev/null || exit @@ -884,6 +940,10 @@ if [[ "${recreate_database:-}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" fi +if [[ "${recover_database:-}" == true ]]; then + database_recovery +fi + # Move possibly existing legacy files to the gravity database migrate_to_database From 469c179b328f93130fb7ccb5334585b15b94dc55 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 11 Sep 2021 21:54:42 +0200 Subject: [PATCH 2/7] Return early from recovery routine when integrity checks didn't show any database errors. Signed-off-by: DL6ER --- gravity.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 911242f7..2d9b5dee 100755 --- a/gravity.sh +++ b/gravity.sh @@ -853,15 +853,16 @@ database_recovery() { echo -ne " ${INFO} ${str}..." if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - no errors found" - else - echo -e "${OVER} ${CROSS} ${str} - errors found:" - while IFS= read -r line ; do echo " - $line"; done <<< "$result" - fi - str="Checking foreign keys of existing gravity database" - echo -ne " ${INFO} ${str}..." - if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"; then - echo -e "${OVER} ${TICK} ${str} - no errors found" + str="Checking foreign keys of existing gravity database" + echo -ne " ${INFO} ${str}..." + if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"; then + echo -e "${OVER} ${TICK} ${str} - no errors found" + return + else + echo -e "${OVER} ${CROSS} ${str} - errors found:" + while IFS= read -r line ; do echo " - $line"; done <<< "$result" + fi else echo -e "${OVER} ${CROSS} ${str} - errors found:" while IFS= read -r line ; do echo " - $line"; done <<< "$result" From ab4bce4787093428305647ad547ed61c8d4809fd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 11 Sep 2021 21:56:44 +0200 Subject: [PATCH 3/7] Allow users to force recovery even when checks are okay using "pihole -g -r recover force" Signed-off-by: DL6ER --- gravity.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 2d9b5dee..c2033fcf 100755 --- a/gravity.sh +++ b/gravity.sh @@ -858,7 +858,9 @@ database_recovery() { echo -ne " ${INFO} ${str}..." if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - no errors found" - return + if [[ $1 != "force" ]]; then + return + fi else echo -e "${OVER} ${CROSS} ${str} - errors found:" while IFS= read -r line ; do echo " - $line"; done <<< "$result" @@ -942,7 +944,7 @@ if [[ "${recreate_database:-}" == true ]]; then fi if [[ "${recover_database:-}" == true ]]; then - database_recovery + database_recovery "$4" fi # Move possibly existing legacy files to the gravity database From b06efb6ab7c3400b99cd7e4f8509cf52092d3671 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Dec 2021 14:00:46 +0100 Subject: [PATCH 4/7] Declare variables local Signed-off-by: DL6ER --- gravity.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index c2033fcf..bd2ae909 100755 --- a/gravity.sh +++ b/gravity.sh @@ -849,7 +849,8 @@ gravity_Cleanup() { } database_recovery() { - str="Checking integrity of existing gravity database" + local result + local str="Checking integrity of existing gravity database" echo -ne " ${INFO} ${str}..." if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - no errors found" From 7167e6d5e42869f8ab370565ae4517cef3b6821e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 21 Dec 2021 16:20:02 +0100 Subject: [PATCH 5/7] Apply suggestions from code review Co-authored-by: Dan Schaper --- gravity.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index bd2ae909..b1ecdc08 100755 --- a/gravity.sh +++ b/gravity.sh @@ -851,6 +851,7 @@ gravity_Cleanup() { database_recovery() { local result local str="Checking integrity of existing gravity database" + local option="${1}" echo -ne " ${INFO} ${str}..." if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - no errors found" @@ -859,7 +860,7 @@ database_recovery() { echo -ne " ${INFO} ${str}..." if result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - no errors found" - if [[ $1 != "force" ]]; then + if [[ "${option}" != "force" ]]; then return fi else From 2e1ce7fc87a5934c9316db1ce1824ac0e3636dcd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Dec 2021 19:52:08 +0100 Subject: [PATCH 6/7] Apply suggestions from code review Co-authored-by: yubiuser --- gravity.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gravity.sh b/gravity.sh index b1ecdc08..385ec3de 100755 --- a/gravity.sh +++ b/gravity.sh @@ -880,6 +880,8 @@ database_recovery() { echo -e "${OVER} ${TICK} ${str} - success" mv "${gravityDBfile}" "${gravityDBfile}.old" mv "${gravityDBfile}.recovered" "${gravityDBfile}" + echo -ne " ${INFO} ${gravityDBfile} has been recovered" + echo -ne " ${INFO} The old ${gravityDBfile} has been moved to ${gravityDBfile}.old" else echo -e "${OVER} ${CROSS} ${str} - the following errors happened:" while IFS= read -r line ; do echo " - $line"; done <<< "$result" From d2a98ae95498d6f72d0b1ceee3d46ec2e7ecfe4c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Dec 2021 19:53:52 +0100 Subject: [PATCH 7/7] Document -r recover force case Signed-off-by: DL6ER --- gravity.sh | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 385ec3de..4fcc281a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -909,14 +909,21 @@ repairSelector() { Attempt to repair gravity database Available options: - pihole -g -r recover Try to recover a damaged gravity database file. - Pi-hole tries to restore as much as possible - from a corrupted gravity database. - pihole -g -r recreate Create a new gravity database file from scratch. - This will remove your existing gravity database - and create a new file from scratch. If you still - have the migration backup created when migrating - to Pi-hole v5.0, Pi-hole will import these files." + pihole -g -r recover Try to recover a damaged gravity database file. + Pi-hole tries to restore as much as possible + from a corrupted gravity database. + + pihole -g -r recover force Pi-hole will run the recovery process even when + no damage is detected. This option is meant to be + a last resort. Recovery is a fragile task + consuming a lot of resources and shouldn't be + performed unnecessarily. + + pihole -g -r recreate Create a new gravity database file from scratch. + This will remove your existing gravity database + and create a new file from scratch. If you still + have the migration backup created when migrating + to Pi-hole v5.0, Pi-hole will import these files." exit 0;; esac }