Make IDs of anti-/gravity lists available in vw_(anti)gravity (#5526)

advanced/Scripts/database_migration/gravity-db.sh

@@ -18,6 +18,11 @@ upgrade_gravityDB(){
 	piholeDir="${2}"
 	auditFile="${piholeDir}/auditlog.list"
 
+	# Exit early if the database does not exist (e.g. in CI tests)
+	if [[ ! -f "${database}" ]]; then
+		return
+	fi
+
 	# Get database version
 	version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
 
@@ -141,4 +146,17 @@ upgrade_gravityDB(){
 		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql"
 		version=17
 	fi
+	if [[ "$version" == "17" ]]; then
+		# Add adlist.id to vw_gravity and vw_antigravity
+		echo -e "  ${INFO} Upgrading gravity database from version 17 to 18"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql"
+		version=18
+	fi
+	if [[ "$version" == "18" ]]; then
+		# Modify DELETE triggers to delete BEFORE instead of AFTER to prevent
+		# foreign key constraint violations
+		echo -e "  ${INFO} Upgrading gravity database from version 18 to 19"
+		pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/18_to_19.sql"
+		version=19
+	fi
 }

advanced/Scripts/database_migration/gravity/17_to_18.sql

@@ -0,0 +1,25 @@
+.timeout 30000
+
+PRAGMA FOREIGN_KEYS=OFF;
+
+BEGIN TRANSACTION;
+
+DROP VIEW vw_gravity;
+CREATE VIEW vw_gravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id
+    FROM gravity
+    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id
+    LEFT JOIN adlist ON adlist.id = gravity.adlist_id
+    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
+    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1);
+
+DROP VIEW vw_antigravity;
+CREATE VIEW vw_antigravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id
+    FROM antigravity
+    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id
+    LEFT JOIN adlist ON adlist.id = antigravity.adlist_id
+    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
+    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1;
+
+UPDATE info SET value = 18 WHERE property = 'version';
+
+COMMIT;

advanced/Scripts/database_migration/gravity/18_to_19.sql

@@ -0,0 +1,27 @@
+.timeout 30000
+
+PRAGMA FOREIGN_KEYS=OFF;
+
+BEGIN TRANSACTION;
+
+DROP TRIGGER tr_domainlist_delete;
+CREATE TRIGGER tr_domainlist_delete BEFORE DELETE ON domainlist
+    BEGIN
+      DELETE FROM domainlist_by_group WHERE domainlist_id = OLD.id;
+    END;
+
+DROP TRIGGER tr_adlist_delete;
+CREATE TRIGGER tr_adlist_delete BEFORE DELETE ON adlist
+    BEGIN
+      DELETE FROM adlist_by_group WHERE adlist_id = OLD.id;
+    END;
+
+DROP TRIGGER tr_client_delete;
+CREATE TRIGGER tr_client_delete BEFORE DELETE ON client
+    BEGIN
+      DELETE FROM client_by_group WHERE client_id = OLD.id;
+    END;
+
+UPDATE info SET value = 19 WHERE property = 'version';
+
+COMMIT;

advanced/Templates/gravity.db.sql

@@ -65,7 +65,7 @@ CREATE TABLE info
 	value TEXT NOT NULL
 );
 
-INSERT INTO "info" VALUES('version','17');
+INSERT INTO "info" VALUES('version','18');
 
 CREATE TABLE domain_audit
 (
@@ -144,14 +144,14 @@ CREATE VIEW vw_regex_blacklist AS SELECT domain, domainlist.id AS id, domainlist
     AND domainlist.type = 3
     ORDER BY domainlist.id;
 
-CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id
+CREATE VIEW vw_gravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id
     FROM gravity
     LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id
     LEFT JOIN adlist ON adlist.id = gravity.adlist_id
     LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
     WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1);
 
-CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id
+CREATE VIEW vw_antigravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id
     FROM antigravity
     LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id
     LEFT JOIN adlist ON adlist.id = antigravity.adlist_id

automated install/basic-install.sh

@@ -2242,6 +2242,11 @@ main() {
     # but before starting or restarting the ftl service
     disable_resolved_stublistener
 
+    # Check if gravity database needs to be upgraded. If so, do it without rebuilding
+    # gravity altogether. This may be a very long running task needlessly blocking
+    # the update process.
+    /opt/pihole/gravity.sh --upgrade
+
     printf "  %b Restarting services...\\n" "${INFO}"
     # Start services
 

gravity.sh

@@ -831,6 +831,7 @@ for var in "$@"; do
   case "${var}" in
     "-f" | "--force" ) forceDelete=true;;
     "-r" | "--repair" ) repairSelector "$3";;
+    "-u" | "--upgrade" ) upgrade_gravityDB "${gravityDBfile}" "${piholeDir}"; exit 0;;
     "-h" | "--help" ) helpFunc;;
   esac
 done