Create new table + view regex_whitelist + rename old regex table to regex_blacklist. This updates the gravity.db version to 3.

Signed-off-by: DL6ER <dl6er@dl6er.de>

advanced/Scripts/database_migration/gravity-db.sh

@@ -29,4 +29,11 @@ upgrade_gravityDB(){
 			database_table_from_file "domain_audit" "${auditFile}"
 		fi
 	fi
+	if [[ "$version" == "2" ]]; then
+		# This migration script upgrades the gravity.db file by
+		# renaming the regex table to regex_blacklist, and
+		# creating a new regex_whitelist table + corresponding linking table and views
+		sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/2_to_3.sql"
+		version=3
+	fi
 }

advanced/Scripts/database_migration/gravity/2_to_3.sql

@@ -0,0 +1,30 @@
+PRAGMA FOREIGN_KEYS=OFF;
+
+ALTER TABLE regex RENAME TO regex_blacklist;
+
+CREATE TABLE regex_blacklist_by_group
+(
+	regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (regex_blacklist_id, group_id)
+);
+
+INSERT INTO regex_blacklist_by_group SELECT * FROM regex_by_group;
+DROP TABLE regex_by_group;
+DROP VIEW vw_regex;
+DROP TRIGGER tr_regex_update;
+
+CREATE VIEW vw_regex_blacklist AS SELECT DISTINCT domain
+    FROM regex
+    LEFT JOIN regex_blacklist_by_group ON regex_blacklist_by_group.regex_blacklist_id = regex_blacklist.id
+    LEFT JOIN "group" ON "group".id = regex_blacklist_by_group.group_id
+    WHERE regex_blacklist.enabled = 1 AND (regex_blacklist_by_group.group_id IS NULL OR "group".enabled = 1)
+    ORDER BY regex_blacklist.id;
+
+CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist
+    BEGIN
+      UPDATE regex_blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+    END;
+
+
+UPDATE info SET value = 3 WHERE property = 'version';

advanced/Scripts/list.sh

@@ -32,12 +32,12 @@ helpFunc() {
     if [[ "${listType}" == "whitelist" ]]; then
         param="w"
         type="whitelist"
-    elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then
+    elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then
         param="-wild"
         type="wildcard blacklist"
-    elif [[ "${listType}" == "regex" ]]; then
+    elif [[ "${listType}" == "regex_blacklist" ]]; then
         param="-regex"
-        type="regex filter"
+        type="regex blacklist filter"
     else
         param="b"
         type="blacklist"
@@ -58,7 +58,8 @@ Options:
   exit 0
 }
 
-EscapeRegexp() {
+Escape
+Regexp() {
     # This way we may safely insert an arbitrary
     # string in our regular expressions
     # This sed is intentionally executed in three steps to ease maintainability
@@ -72,7 +73,7 @@ HandleOther() {
 
     # Check validity of domain (don't check for regex entries)
     if [[ "${#domain}" -le 253 ]]; then
-        if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then
+        if [[ "${listType}" == "regex_blacklist" && "${wildcard}" == false ]]; then
             validDomain="${domain}"
         else
             validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
@@ -88,9 +89,9 @@ HandleOther() {
 }
 
 ProcessDomainList() {
-    if [[ "${listType}" == "regex" ]]; then
+    if [[ "${listType}" == "regex_blacklist" ]]; then
         # Regex filter list
-        listname="regex filters"
+        listname="regex blacklist filters"
     else
         # Whitelist / Blacklist
         listname="${listType}"
@@ -106,7 +107,7 @@ ProcessDomainList() {
         # if delmode then remove from desired list but do not add to the other
         if ${addmode}; then
             AddDomain "${dom}" "${listType}"
-            if [[ ! "${listType}" == "regex" ]]; then
+            if [[ ! "${listType}" == "regex_blacklist" ]]; then
                 RemoveDomain "${dom}" "${listAlt}"
             fi
         else
@@ -215,8 +216,8 @@ for var in "$@"; do
     case "${var}" in
         "-w" | "whitelist"   ) listType="whitelist"; listAlt="blacklist";;
         "-b" | "blacklist"   ) listType="blacklist"; listAlt="whitelist";;
-        "--wild" | "wildcard" ) listType="regex"; wildcard=true;;
-        "--regex" | "regex"   ) listType="regex";;
+        "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;;
+        "--regex" | "regex"   ) listType="regex_blacklist";;
         "-nr"| "--noreload"  ) reload=false;;
         "-d" | "--delmode"   ) addmode=false;;
         "-q" | "--quiet"     ) verbose=false;;

advanced/Scripts/piholeDebug.sh

@@ -1088,8 +1088,12 @@ show_blacklist() {
     show_db_entries "Blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50"
 }
 
-show_regexlist() {
-    show_db_entries "Regexlist" "SELECT * FROM regex" "4 100 7 10 13 50"
+show_regexblacklist() {
+    show_db_entries "Regexblacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50"
+}
+
+show_regexwhitelist() {
+    show_db_entries "Regexwhitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50"
 }
 
 analyze_gravity_list() {
@@ -1268,7 +1272,8 @@ analyze_gravity_list
 show_adlists
 show_whitelist
 show_blacklist
-show_regexlist
+show_regexblacklist
+show_regexwhitelist
 show_content_of_pihole_files
 parse_locale
 analyze_pihole_log

advanced/Scripts/query.sh

@@ -133,7 +133,7 @@ scanDatabaseTable "${domainQuery}" "whitelist" "${exact}"
 scanDatabaseTable "${domainQuery}" "blacklist" "${exact}"
 
 # Scan Regex table
-mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null)
+mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_blacklist" 2> /dev/null)
 
 # If we have regexps to process
 if [[ "${#regexList[@]}" -ne 0 ]]; then

gravity.sh

@@ -183,7 +183,7 @@ migrate_to_database() {
     if [ -e "${regexFile}" ]; then
       # Store regex domains in database
       echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
-      database_table_from_file "regex" "${regexFile}"
+      database_table_from_file "regex_blacklist" "${regexFile}"
     fi
   fi
 
@@ -591,9 +591,10 @@ gravity_Table_Count() {
 
 # Output count of blacklisted domains and regex filters
 gravity_ShowCount() {
-  gravity_Table_Count "blacklist" "blacklisted domains"
-  gravity_Table_Count "whitelist" "whitelisted domains"
-  gravity_Table_Count "regex" "regex filters"
+  gravity_Table_Count "blacklist" "exact blacklisted domains"
+  gravity_Table_Count "regex_blacklist" "regex blacklist filters"
+  gravity_Table_Count "whitelist" "exact whitelisted domains"
+  gravity_Table_Count "regex_whitelist" "regex whitelist filters"
 }
 
 # Parse list of domains into hosts format