Merge pull request #2803 from pi-hole/new/gravity.db_domain_groups

Implement groups for lists

advanced/Scripts/piholeDebug.sh

@@ -1077,7 +1077,7 @@ show_db_entries() {
 }
 
 show_adlists() {
-    show_db_entries "Adlists" "SELECT * FROM adlists" "4 100 7 10 13 50"
+    show_db_entries "Adlists" "SELECT * FROM adlist" "4 100 7 10 13 50"
 }
 
 show_whitelist() {

advanced/Scripts/query.sh

@@ -201,7 +201,7 @@ fi
 # Get adlist file content as array
 if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
     # Retrieve source URLs from gravity database
-    mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)"
+    mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
 fi
 
 # Print "Exact matches for" title

advanced/Scripts/webpage.sh

@@ -400,13 +400,13 @@ CustomizeAdLists() {
     address="${args[3]}"
 
     if [[ "${args[2]}" == "enable" ]]; then
-        sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 1 WHERE address = '${address}'"
+        sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'"
     elif [[ "${args[2]}" == "disable" ]]; then
-        sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 0 WHERE address = '${address}'"
+        sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'"
     elif [[ "${args[2]}" == "add" ]]; then
-        sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlists (address) VALUES ('${address}')"
+        sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address) VALUES ('${address}')"
     elif [[ "${args[2]}" == "del" ]]; then
-        sqlite3 "${gravityDBfile}" "DELETE FROM adlists WHERE address = '${address}'"
+        sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'"
     else
         echo "Not permitted"
         return 1

advanced/Templates/gravity.db.sql

@@ -1,3 +1,13 @@
+PRAGMA FOREIGN_KEYS=ON;
+
+CREATE TABLE "group"
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	enabled BOOLEAN NOT NULL DEFAULT 1,
+	name TEXT NOT NULL,
+	description TEXT
+);
+
 CREATE TABLE whitelist
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -7,6 +17,14 @@ CREATE TABLE whitelist
 	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
 	comment TEXT
 );
+
+CREATE TABLE whitelist_by_group
+(
+	whitelist_id INTEGER NOT NULL REFERENCES whitelist (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (whitelist_id, group_id)
+);
+
 CREATE TABLE blacklist
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -16,6 +34,14 @@ CREATE TABLE blacklist
 	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
 	comment TEXT
 );
+
+CREATE TABLE blacklist_by_group
+(
+	blacklist_id INTEGER NOT NULL REFERENCES blacklist (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (blacklist_id, group_id)
+);
+
 CREATE TABLE regex
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -25,7 +51,15 @@ CREATE TABLE regex
 	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
 	comment TEXT
 );
-CREATE TABLE adlists
+
+CREATE TABLE regex_by_group
+(
+	regex_id INTEGER NOT NULL REFERENCES regex (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (regex_id, group_id)
+);
+
+CREATE TABLE adlist
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
 	address TEXT UNIQUE NOT NULL,
@@ -34,6 +68,14 @@ CREATE TABLE adlists
 	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
 	comment TEXT
 );
+
+CREATE TABLE adlist_by_group
+(
+	adlist_id INTEGER NOT NULL REFERENCES adlist (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (adlist_id, group_id)
+);
+
 CREATE TABLE gravity
 (
 	domain TEXT PRIMARY KEY
@@ -46,47 +88,55 @@ CREATE TABLE info
 
 INSERT INTO info VALUES("version","1");
 
-CREATE VIEW vw_gravity AS SELECT a.domain
-    FROM gravity a
-    WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1);
+CREATE VIEW vw_gravity AS SELECT domain
+    FROM gravity
+    WHERE domain NOT IN (SELECT domain from vw_whitelist);
 
-CREATE VIEW vw_whitelist AS SELECT a.domain
-    FROM whitelist a
-    WHERE a.enabled == 1
-    ORDER BY a.id;
+CREATE VIEW vw_whitelist AS SELECT DISTINCT domain
+    FROM whitelist
+    LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id
+    LEFT JOIN "group" ON "group".id = whitelist_by_group.group_id
+    WHERE whitelist.enabled = 1 AND (whitelist_by_group.group_id IS NULL OR "group".enabled = 1)
+    ORDER BY whitelist.id;
 
 CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist
     BEGIN
       UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
     END;
 
-CREATE VIEW vw_blacklist AS SELECT a.domain
-    FROM blacklist a
-    WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist
-    ORDER BY a.id;
+CREATE VIEW vw_blacklist AS SELECT DISTINCT domain
+    FROM blacklist
+    LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id
+    LEFT JOIN "group" ON "group".id = blacklist_by_group.group_id
+    WHERE blacklist.enabled = 1 AND (blacklist_by_group.group_id IS NULL OR "group".enabled = 1)
+    ORDER BY blacklist.id;
 
 CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist
     BEGIN
       UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
     END;
 
-CREATE VIEW vw_regex AS SELECT a.domain
-    FROM regex a
-    WHERE a.enabled == 1
-    ORDER BY a.id;
+CREATE VIEW vw_regex AS SELECT DISTINCT domain
+    FROM regex
+    LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id
+    LEFT JOIN "group" ON "group".id = regex_by_group.group_id
+    WHERE regex.enabled = 1 AND (regex_by_group.group_id IS NULL OR "group".enabled = 1)
+    ORDER BY regex.id;
 
 CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex
     BEGIN
       UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
     END;
 
-CREATE VIEW vw_adlists AS SELECT a.address
-    FROM adlists a
-    WHERE a.enabled == 1
-    ORDER BY a.id;
+CREATE VIEW vw_adlist AS SELECT DISTINCT address
+    FROM adlist
+    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id
+    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
+    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    ORDER BY adlist.id;
 
-CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists
+CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist
     BEGIN
-      UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address;
+      UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address;
     END;
 

gravity.sh

@@ -111,7 +111,7 @@ database_table_from_file() {
     # No need to modify the input data for the gravity table
     inputfile="${source}"
   else
-    # Apply format for white-, blacklist, regex, and adlists tables
+    # Apply format for white-, blacklist, regex, and adlist tables
     local rowid
     declare -i rowid
     rowid=1
@@ -159,9 +159,9 @@ migrate_to_database() {
 
   # Migrate list files to new database
   if [[ -e "${adListFile}" ]]; then
-    # Store adlists domains in database
+    # Store adlist domains in database
     echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
-    database_table_from_file "adlists" "${adListFile}"
+    database_table_from_file "adlist" "${adListFile}"
   fi
   if [[ -e "${blacklistFile}" ]]; then
     # Store blacklisted domains in database
@@ -236,13 +236,13 @@ gravity_CheckDNSResolutionAvailable() {
   gravity_CheckDNSResolutionAvailable
 }
 
-# Retrieve blocklist URLs and parse domains from adlists.list
+# Retrieve blocklist URLs and parse domains from adlist.list
 gravity_GetBlocklistUrls() {
   echo -e "  ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
 
   # Retrieve source URLs from gravity database
   # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true)
-  mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)"
+  mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)"
 
   # Parse source domains from $sources
   mapfile -t sourceDomains <<< "$(