Merge branch 'development-v6' into development-v6-merge-development

Signed-off-by: DL6ER <dl6er@dl6er.de>

advanced/Scripts/database_migration/gravity-db.sh

@@ -18,6 +18,11 @@ upgrade_gravityDB(){
 	piholeDir="${2}"
 	auditFile="${piholeDir}/auditlog.list"
 
+	# Exit early if the database does not exist (e.g. in CI tests)
+	if [[ ! -f "${database}" ]]; then
+		return
+	fi
+
 	# Get database version
 	version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
 
@@ -141,4 +146,17 @@ upgrade_gravityDB(){
 		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql"
 		version=17
 	fi
+	if [[ "$version" == "17" ]]; then
+		# Add adlist.id to vw_gravity and vw_antigravity
+		echo -e "  ${INFO} Upgrading gravity database from version 17 to 18"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql"
+		version=18
+	fi
+	if [[ "$version" == "18" ]]; then
+		# Modify DELETE triggers to delete BEFORE instead of AFTER to prevent
+		# foreign key constraint violations
+		echo -e "  ${INFO} Upgrading gravity database from version 18 to 19"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/18_to_19.sql"
+		version=19
+	fi
 }

advanced/Scripts/database_migration/gravity/17_to_18.sql

@@ -0,0 +1,25 @@
+.timeout 30000
+
+PRAGMA FOREIGN_KEYS=OFF;
+
+BEGIN TRANSACTION;
+
+DROP VIEW vw_gravity;
+CREATE VIEW vw_gravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id
+    FROM gravity
+    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id
+    LEFT JOIN adlist ON adlist.id = gravity.adlist_id
+    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
+    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1);
+
+DROP VIEW vw_antigravity;
+CREATE VIEW vw_antigravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id
+    FROM antigravity
+    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id
+    LEFT JOIN adlist ON adlist.id = antigravity.adlist_id
+    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
+    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1;
+
+UPDATE info SET value = 18 WHERE property = 'version';
+
+COMMIT;

advanced/Scripts/database_migration/gravity/18_to_19.sql

@@ -0,0 +1,27 @@
+.timeout 30000
+
+PRAGMA FOREIGN_KEYS=OFF;
+
+BEGIN TRANSACTION;
+
+DROP TRIGGER tr_domainlist_delete;
+CREATE TRIGGER tr_domainlist_delete BEFORE DELETE ON domainlist
+    BEGIN
+      DELETE FROM domainlist_by_group WHERE domainlist_id = OLD.id;
+    END;
+
+DROP TRIGGER tr_adlist_delete;
+CREATE TRIGGER tr_adlist_delete BEFORE DELETE ON adlist
+    BEGIN
+      DELETE FROM adlist_by_group WHERE adlist_id = OLD.id;
+    END;
+
+DROP TRIGGER tr_client_delete;
+CREATE TRIGGER tr_client_delete BEFORE DELETE ON client
+    BEGIN
+      DELETE FROM client_by_group WHERE client_id = OLD.id;
+    END;
+
+UPDATE info SET value = 19 WHERE property = 'version';
+
+COMMIT;

advanced/Scripts/piholeDebug.sh

@@ -74,7 +74,6 @@ PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole"
 
 PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log"
 PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*"
-PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list"
 PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate"
 PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml"
 PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions"

advanced/Templates/gravity.db.sql

@@ -65,7 +65,7 @@ CREATE TABLE info
 	value TEXT NOT NULL
 );
 
-INSERT INTO "info" VALUES('version','17');
+INSERT INTO "info" VALUES('version','18');
 
 CREATE TABLE domain_audit
 (
@@ -144,14 +144,14 @@ CREATE VIEW vw_regex_blacklist AS SELECT domain, domainlist.id AS id, domainlist
     AND domainlist.type = 3
     ORDER BY domainlist.id;
 
-CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id
+CREATE VIEW vw_gravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id
     FROM gravity
     LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id
     LEFT JOIN adlist ON adlist.id = gravity.adlist_id
     LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
     WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1);
 
-CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id
+CREATE VIEW vw_antigravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id
     FROM antigravity
     LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id
     LEFT JOIN adlist ON adlist.id = antigravity.adlist_id

automated install/basic-install.sh

@@ -2260,6 +2260,11 @@ main() {
     # but before starting or resttarting the ftl service
     disable_resolved_stublistener
 
+    # Check if gravity database needs to be upgraded. If so, do it without rebuilding
+    # gravity altogether. This may be a very long running task needlessly blocking
+    # the update process.
+    /opt/pihole/gravity.sh --upgrade
+
     printf "  %b Restarting services...\\n" "${INFO}"
     # Start services
 

gravity.sh

@@ -36,9 +36,6 @@ blacklistFile="${piholeDir}/blacklist.txt"
 regexFile="${piholeDir}/regex.list"
 adListFile="${piholeDir}/adlists.list"
 
-localList="${piholeDir}/local.list"
-VPNList="/etc/openvpn/ipp.txt"
-
 piholeGitDir="/etc/.pihole"
 GRAVITYDB=$(getFTLConfigValue files.gravity)
 GRAVITY_TMPDIR=$(getFTLConfigValue files.gravity_tmp)
@@ -296,12 +293,7 @@ migrate_to_database() {
 
 # Determine if DNS resolution is available before proceeding
 gravity_CheckDNSResolutionAvailable() {
-  local lookupDomain="pi.hole"
-
-  # Determine if $localList does not exist, and ensure it is not empty
-  if [[ ! -e "${localList}" ]] || [[ -s "${localList}" ]]; then
-    lookupDomain="raw.githubusercontent.com"
-  fi
+  local lookupDomain="raw.githubusercontent.com"
 
   # Determine if $lookupDomain is resolvable
   if timeout 4 getent hosts "${lookupDomain}" &>/dev/null; then
@@ -697,18 +689,6 @@ gravity_ShowCount() {
   gravity_Table_Count "vw_regex_whitelist" "regex allowed filters"
 }
 
-# Create "localhost" entries into hosts format
-gravity_generateLocalList() {
-  # Empty $localList if it already exists, otherwise, create it
-  echo "### Do not modify this file, it will be overwritten by pihole -g" >"${localList}"
-  chmod 644 "${localList}"
-
-  # Add additional LAN hosts provided by OpenVPN (if available)
-  if [[ -f "${VPNList}" ]]; then
-    awk -F, '{printf $2"\t"$1".vpn\n"}' "${VPNList}" >>"${localList}"
-  fi
-}
-
 # Trap Ctrl-C
 gravity_Trap() {
   trap '{ echo -e "\\n\\n  ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
@@ -844,9 +824,10 @@ Available options:
 
 for var in "$@"; do
   case "${var}" in
-  "-f" | "--force") forceDelete=true ;;
-  "-r" | "--repair") repairSelector "$3" ;;
-  "-h" | "--help") helpFunc ;;
+  "-f" | "--force" ) forceDelete=true;;
+  "-r" | "--repair" ) repairSelector "$3";;
+  "-u" | "--upgrade" ) upgrade_gravityDB "${gravityDBfile}" "${piholeDir}"; exit 0;;
+  "-h" | "--help" ) helpFunc;;
   esac
 done
 
@@ -897,9 +878,6 @@ if ! gravity_DownloadBlocklists; then
   exit 1
 fi
 
-# Create local.list
-gravity_generateLocalList
-
 # Update gravity timestamp
 update_gravity_timestamp