From 054c7a2c050b416ac7de09e29351bdb69564430d Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 8 Jul 2019 21:23:46 +0200
Subject: [PATCH 01/24] Create new table + view regex_whitelist + rename old
 regex table to regex_blacklist. This updates the gravity.db version to 3.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 .../Scripts/database_migration/gravity-db.sh  |  7 +++++
 .../database_migration/gravity/2_to_3.sql     | 30 +++++++++++++++++++
 advanced/Scripts/list.sh                      | 21 ++++++-------
 advanced/Scripts/piholeDebug.sh               | 11 +++++--
 advanced/Scripts/query.sh                     |  2 +-
 gravity.sh                                    |  9 +++---
 6 files changed, 62 insertions(+), 18 deletions(-)
 create mode 100644 advanced/Scripts/database_migration/gravity/2_to_3.sql

diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh
index 51a3480b..f37ce176 100644
--- a/advanced/Scripts/database_migration/gravity-db.sh
+++ b/advanced/Scripts/database_migration/gravity-db.sh
@@ -29,4 +29,11 @@ upgrade_gravityDB(){
 			database_table_from_file "domain_audit" "${auditFile}"
 		fi
 	fi
+	if [[ "$version" == "2" ]]; then
+		# This migration script upgrades the gravity.db file by
+		# renaming the regex table to regex_blacklist, and
+		# creating a new regex_whitelist table + corresponding linking table and views
+		sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/2_to_3.sql"
+		version=3
+	fi
 }
diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql
new file mode 100644
index 00000000..e368592a
--- /dev/null
+++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql
@@ -0,0 +1,30 @@
+PRAGMA FOREIGN_KEYS=OFF;
+
+ALTER TABLE regex RENAME TO regex_blacklist;
+
+CREATE TABLE regex_blacklist_by_group
+(
+	regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (regex_blacklist_id, group_id)
+);
+
+INSERT INTO regex_blacklist_by_group SELECT * FROM regex_by_group;
+DROP TABLE regex_by_group;
+DROP VIEW vw_regex;
+DROP TRIGGER tr_regex_update;
+
+CREATE VIEW vw_regex_blacklist AS SELECT DISTINCT domain
+    FROM regex
+    LEFT JOIN regex_blacklist_by_group ON regex_blacklist_by_group.regex_blacklist_id = regex_blacklist.id
+    LEFT JOIN "group" ON "group".id = regex_blacklist_by_group.group_id
+    WHERE regex_blacklist.enabled = 1 AND (regex_blacklist_by_group.group_id IS NULL OR "group".enabled = 1)
+    ORDER BY regex_blacklist.id;
+
+CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist
+    BEGIN
+      UPDATE regex_blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+    END;
+
+
+UPDATE info SET value = 3 WHERE property = 'version';
diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index fa81348b..84acf4fc 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -32,12 +32,12 @@ helpFunc() {
     if [[ "${listType}" == "whitelist" ]]; then
         param="w"
         type="whitelist"
-    elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then
+    elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then
         param="-wild"
         type="wildcard blacklist"
-    elif [[ "${listType}" == "regex" ]]; then
+    elif [[ "${listType}" == "regex_blacklist" ]]; then
         param="-regex"
-        type="regex filter"
+        type="regex blacklist filter"
     else
         param="b"
         type="blacklist"
@@ -58,7 +58,8 @@ Options:
   exit 0
 }
 
-EscapeRegexp() {
+Escape
+Regexp() {
     # This way we may safely insert an arbitrary
     # string in our regular expressions
     # This sed is intentionally executed in three steps to ease maintainability
@@ -72,7 +73,7 @@ HandleOther() {
 
     # Check validity of domain (don't check for regex entries)
     if [[ "${#domain}" -le 253 ]]; then
-        if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then
+        if [[ "${listType}" == "regex_blacklist" && "${wildcard}" == false ]]; then
             validDomain="${domain}"
         else
             validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
@@ -88,9 +89,9 @@ HandleOther() {
 }
 
 ProcessDomainList() {
-    if [[ "${listType}" == "regex" ]]; then
+    if [[ "${listType}" == "regex_blacklist" ]]; then
         # Regex filter list
-        listname="regex filters"
+        listname="regex blacklist filters"
     else
         # Whitelist / Blacklist
         listname="${listType}"
@@ -106,7 +107,7 @@ ProcessDomainList() {
         # if delmode then remove from desired list but do not add to the other
         if ${addmode}; then
             AddDomain "${dom}" "${listType}"
-            if [[ ! "${listType}" == "regex" ]]; then
+            if [[ ! "${listType}" == "regex_blacklist" ]]; then
                 RemoveDomain "${dom}" "${listAlt}"
             fi
         else
@@ -215,8 +216,8 @@ for var in "$@"; do
     case "${var}" in
         "-w" | "whitelist"   ) listType="whitelist"; listAlt="blacklist";;
         "-b" | "blacklist"   ) listType="blacklist"; listAlt="whitelist";;
-        "--wild" | "wildcard" ) listType="regex"; wildcard=true;;
-        "--regex" | "regex"   ) listType="regex";;
+        "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;;
+        "--regex" | "regex"   ) listType="regex_blacklist";;
         "-nr"| "--noreload"  ) reload=false;;
         "-d" | "--delmode"   ) addmode=false;;
         "-q" | "--quiet"     ) verbose=false;;
diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index 2bfeef2d..965250f0 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1088,8 +1088,12 @@ show_blacklist() {
     show_db_entries "Blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50"
 }
 
-show_regexlist() {
-    show_db_entries "Regexlist" "SELECT * FROM regex" "4 100 7 10 13 50"
+show_regexblacklist() {
+    show_db_entries "Regexblacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50"
+}
+
+show_regexwhitelist() {
+    show_db_entries "Regexwhitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50"
 }
 
 analyze_gravity_list() {
@@ -1268,7 +1272,8 @@ analyze_gravity_list
 show_adlists
 show_whitelist
 show_blacklist
-show_regexlist
+show_regexblacklist
+show_regexwhitelist
 show_content_of_pihole_files
 parse_locale
 analyze_pihole_log
diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 4fc82744..a587f238 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -133,7 +133,7 @@ scanDatabaseTable "${domainQuery}" "whitelist" "${exact}"
 scanDatabaseTable "${domainQuery}" "blacklist" "${exact}"
 
 # Scan Regex table
-mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null)
+mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_blacklist" 2> /dev/null)
 
 # If we have regexps to process
 if [[ "${#regexList[@]}" -ne 0 ]]; then
diff --git a/gravity.sh b/gravity.sh
index 89f77ce0..a379e248 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -183,7 +183,7 @@ migrate_to_database() {
     if [ -e "${regexFile}" ]; then
       # Store regex domains in database
       echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
-      database_table_from_file "regex" "${regexFile}"
+      database_table_from_file "regex_blacklist" "${regexFile}"
     fi
   fi
 
@@ -591,9 +591,10 @@ gravity_Table_Count() {
 
 # Output count of blacklisted domains and regex filters
 gravity_ShowCount() {
-  gravity_Table_Count "blacklist" "blacklisted domains"
-  gravity_Table_Count "whitelist" "whitelisted domains"
-  gravity_Table_Count "regex" "regex filters"
+  gravity_Table_Count "blacklist" "exact blacklisted domains"
+  gravity_Table_Count "regex_blacklist" "regex blacklist filters"
+  gravity_Table_Count "whitelist" "exact whitelisted domains"
+  gravity_Table_Count "regex_whitelist" "regex whitelist filters"
 }
 
 # Parse list of domains into hosts format

From f5121c64be312a0c4c900383d882f20cccdb8a6d Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 8 Jul 2019 21:39:30 +0200
Subject: [PATCH 02/24] We should still add the regex lines (initially) to the
 regex table as the renaming will happen only after the importing.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gravity.sh b/gravity.sh
index a379e248..f9ecc1d1 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -182,8 +182,10 @@ migrate_to_database() {
     fi
     if [ -e "${regexFile}" ]; then
       # Store regex domains in database
+      # Important note: We need to add the domains to the "regex" table
+      # as it will only later be renamed to "regex_blacklist"!
       echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
-      database_table_from_file "regex_blacklist" "${regexFile}"
+      database_table_from_file "regex" "${regexFile}"
     fi
   fi
 

From 0683842ec30081e57ce2e801ebcfec7c4f08f8ff Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 8 Jul 2019 21:43:49 +0200
Subject: [PATCH 03/24] Fix typo in 2->3 migration script.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/database_migration/gravity/2_to_3.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql
index e368592a..9b0bd412 100644
--- a/advanced/Scripts/database_migration/gravity/2_to_3.sql
+++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql
@@ -15,7 +15,7 @@ DROP VIEW vw_regex;
 DROP TRIGGER tr_regex_update;
 
 CREATE VIEW vw_regex_blacklist AS SELECT DISTINCT domain
-    FROM regex
+    FROM regex_blacklist
     LEFT JOIN regex_blacklist_by_group ON regex_blacklist_by_group.regex_blacklist_id = regex_blacklist.id
     LEFT JOIN "group" ON "group".id = regex_blacklist_by_group.group_id
     WHERE regex_blacklist.enabled = 1 AND (regex_blacklist_by_group.group_id IS NULL OR "group".enabled = 1)

From 87f75c737a8d7eced48c5f61e2bb6581db1f4c2a Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 10 Jul 2019 12:00:38 +0200
Subject: [PATCH 04/24] Review comments.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh        |  3 +--
 advanced/Scripts/piholeDebug.sh | 16 ++++------------
 2 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 84acf4fc..e3dc552a 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -58,8 +58,7 @@ Options:
   exit 0
 }
 
-Escape
-Regexp() {
+EscapeRegexp() {
     # This way we may safely insert an arbitrary
     # string in our regular expressions
     # This sed is intentionally executed in three steps to ease maintainability
diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index 965250f0..0924e984 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1081,19 +1081,13 @@ show_adlists() {
 }
 
 show_whitelist() {
-    show_db_entries "Whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50"
+    show_db_entries "Exact whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50"
+    show_db_entries "Regex whitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50"
 }
 
 show_blacklist() {
-    show_db_entries "Blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50"
-}
-
-show_regexblacklist() {
-    show_db_entries "Regexblacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50"
-}
-
-show_regexwhitelist() {
-    show_db_entries "Regexwhitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50"
+    show_db_entries "Exact blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50"
+    show_db_entries "Regex blacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50"
 }
 
 analyze_gravity_list() {
@@ -1272,8 +1266,6 @@ analyze_gravity_list
 show_adlists
 show_whitelist
 show_blacklist
-show_regexblacklist
-show_regexwhitelist
 show_content_of_pihole_files
 parse_locale
 analyze_pihole_log

From 65fdbc85d5e1cbafb986da221783fc73ad972df8 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 10 Jul 2019 12:01:38 +0200
Subject: [PATCH 05/24] Add timeout to migration script (2->3).

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/database_migration/gravity/2_to_3.sql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql
index 9b0bd412..a2602c4a 100644
--- a/advanced/Scripts/database_migration/gravity/2_to_3.sql
+++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql
@@ -1,3 +1,5 @@
+.timeout 30000
+
 PRAGMA FOREIGN_KEYS=OFF;
 
 ALTER TABLE regex RENAME TO regex_blacklist;

From 96031214c6a0d797015b4461901cf1059edbc00a Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 22 Jul 2019 19:35:16 +0200
Subject: [PATCH 06/24] Add support for whitelist regex filter management via
 CLI.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 .../database_migration/gravity/2_to_3.sql     | 29 +++++++++++++++++++
 advanced/Scripts/list.sh                      | 13 +++++++--
 pihole                                        |  2 ++
 3 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql
index a2602c4a..d7997936 100644
--- a/advanced/Scripts/database_migration/gravity/2_to_3.sql
+++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql
@@ -28,5 +28,34 @@ CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist
       UPDATE regex_blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
     END;
 
+CREATE TABLE regex_whitelist
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	domain TEXT UNIQUE NOT NULL,
+	enabled BOOLEAN NOT NULL DEFAULT 1,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+
+CREATE TABLE regex_whitelist_by_group
+(
+	regex_id INTEGER NOT NULL REFERENCES regex_whitelist (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (regex_id, group_id)
+);
+
+CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain
+    FROM regex_whitelist
+    LEFT JOIN regex_whitelist_by_group ON regex_whitelist_by_group.regex_id = regex_whitelist.id
+    LEFT JOIN "group" ON "group".id = regex_whitelist_by_group.group_id
+    WHERE regex_whitelist.enabled = 1 AND (regex_whitelist_by_group.group_id IS NULL OR "group".enabled = 1)
+    ORDER BY regex_whitelist.id;
+
+CREATE TRIGGER tr_regex_whitelist_update AFTER UPDATE ON regex_whitelist
+    BEGIN
+      UPDATE regex_whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+    END;
+
 
 UPDATE info SET value = 3 WHERE property = 'version';
diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index e3dc552a..4ef86407 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -38,6 +38,9 @@ helpFunc() {
     elif [[ "${listType}" == "regex_blacklist" ]]; then
         param="-regex"
         type="regex blacklist filter"
+    elif [[ "${listType}" == "regex_blacklist" ]]; then
+        param="-whiteregex"
+        type="regex whitelist filter"
     else
         param="b"
         type="blacklist"
@@ -89,8 +92,11 @@ HandleOther() {
 
 ProcessDomainList() {
     if [[ "${listType}" == "regex_blacklist" ]]; then
-        # Regex filter list
+        # Regex black filter list
         listname="regex blacklist filters"
+    elif [[ "${listType}" == "regex_whitelist" ]]; then
+        # Regex white filter list
+        listname="regex whitelist filters"
     else
         # Whitelist / Blacklist
         listname="${listType}"
@@ -106,7 +112,7 @@ ProcessDomainList() {
         # if delmode then remove from desired list but do not add to the other
         if ${addmode}; then
             AddDomain "${dom}" "${listType}"
-            if [[ ! "${listType}" == "regex_blacklist" ]]; then
+            if [[ ! "${listType}" == "regex_"*"list" ]]; then
                 RemoveDomain "${dom}" "${listAlt}"
             fi
         else
@@ -173,7 +179,7 @@ Displaylist() {
     data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)"
 
     if [[ -z $data ]]; then
-        echo -e "Not showing empty ${listname}"
+        echo -e "Not showing empty list"
     else
         echo -e "Displaying ${listname}:"
         count=1
@@ -217,6 +223,7 @@ for var in "$@"; do
         "-b" | "blacklist"   ) listType="blacklist"; listAlt="whitelist";;
         "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;;
         "--regex" | "regex"   ) listType="regex_blacklist";;
+        "--whiteregex" | "whiteregex" ) listType="regex_whitelist";;
         "-nr"| "--noreload"  ) reload=false;;
         "-d" | "--delmode"   ) addmode=false;;
         "-q" | "--quiet"     ) verbose=false;;
diff --git a/pihole b/pihole
index 9fa65a8f..411b5791 100755
--- a/pihole
+++ b/pihole
@@ -377,6 +377,7 @@ Whitelist/Blacklist Options:
   -b, blacklist       Blacklist domain(s)
   --wild, wildcard     Wildcard blacklist domain(s)
   --regex, regex       Regex blacklist domains(s)
+  --whiteregex         Regex whitelist domains(s)
                         Add '-h' for more info on whitelist/blacklist usage
 
 Debugging Options:
@@ -438,6 +439,7 @@ case "${1}" in
   "-b" | "blacklist"            ) listFunc "$@";;
   "--wild" | "wildcard"         ) listFunc "$@";;
   "--regex" | "regex"           ) listFunc "$@";;
+  "--whiteregex" | "whiteregex" ) listFunc "$@";;
   "-d" | "debug"                ) debugFunc "$@";;
   "-f" | "flush"                ) flushFunc "$@";;
   "-up" | "updatePihole"        ) updatePiholeFunc "$@";;

From 0d28dce326facddc13f02fe80903f78ea732736a Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 22 Jul 2019 20:18:15 +0200
Subject: [PATCH 07/24] Print group table contents in debug log.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/piholeDebug.sh | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index 0924e984..b2533eb0 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1076,18 +1076,27 @@ show_db_entries() {
     IFS="$OLD_IFS"
 }
 
+show_groups() {
+    show_db_entries "Groups" "SELECT * FROM \"group\"" "4 100 7 10 13 50"
+}
+
 show_adlists() {
     show_db_entries "Adlists" "SELECT * FROM adlist" "4 100 7 10 13 50"
+    show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 100 7 10 13 50"
 }
 
 show_whitelist() {
     show_db_entries "Exact whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50"
     show_db_entries "Regex whitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50"
+    show_db_entries "Exact whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 100 7 10 13 50"
+    show_db_entries "Regex whitelist groups" "SELECT * FROM whitelist_by_group" "4 100 7 10 13 50"
 }
 
 show_blacklist() {
     show_db_entries "Exact blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50"
     show_db_entries "Regex blacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50"
+    show_db_entries "Exact blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 100 7 10 13 50"
+    show_db_entries "Regex blacklist groups" "SELECT * FROM blacklist_by_group" "4 100 7 10 13 50"
 }
 
 analyze_gravity_list() {
@@ -1263,6 +1272,7 @@ process_status
 parse_setup_vars
 check_x_headers
 analyze_gravity_list
+show_groups
 show_adlists
 show_whitelist
 show_blacklist

From 0692be9bae12a8a96f582a2dc15968282df2e8f2 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 22 Jul 2019 20:59:52 +0200
Subject: [PATCH 08/24] Fix small mistake in 2->3 upgrade script.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/database_migration/gravity/2_to_3.sql | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql
index d7997936..e121f28c 100644
--- a/advanced/Scripts/database_migration/gravity/2_to_3.sql
+++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql
@@ -40,14 +40,14 @@ CREATE TABLE regex_whitelist
 
 CREATE TABLE regex_whitelist_by_group
 (
-	regex_id INTEGER NOT NULL REFERENCES regex_whitelist (id),
+	regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id),
 	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (regex_id, group_id)
+	PRIMARY KEY (regex_whitelist_id, group_id)
 );
 
 CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain
     FROM regex_whitelist
-    LEFT JOIN regex_whitelist_by_group ON regex_whitelist_by_group.regex_id = regex_whitelist.id
+    LEFT JOIN regex_whitelist_by_group ON regex_whitelist_by_group.regex_whitelist_id = regex_whitelist.id
     LEFT JOIN "group" ON "group".id = regex_whitelist_by_group.group_id
     WHERE regex_whitelist.enabled = 1 AND (regex_whitelist_by_group.group_id IS NULL OR "group".enabled = 1)
     ORDER BY regex_whitelist.id;

From 40d0caa70b642382b2ca35344e6da7fd6c452432 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 22 Jul 2019 21:03:42 +0200
Subject: [PATCH 09/24] Add undocumented --whitewild option that does the same
 --wild does for the whitelist.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 3 ++-
 pihole                   | 1 +
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 4ef86407..0183a9e2 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -75,7 +75,7 @@ HandleOther() {
 
     # Check validity of domain (don't check for regex entries)
     if [[ "${#domain}" -le 253 ]]; then
-        if [[ "${listType}" == "regex_blacklist" && "${wildcard}" == false ]]; then
+        if [[ ( "${listType}" == "regex_blacklist" || "${listType}" == "regex_whitelist" ) && "${wildcard}" == false ]]; then
             validDomain="${domain}"
         else
             validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
@@ -224,6 +224,7 @@ for var in "$@"; do
         "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;;
         "--regex" | "regex"   ) listType="regex_blacklist";;
         "--whiteregex" | "whiteregex" ) listType="regex_whitelist";;
+        "--whitewild" | "whitewild" ) listType="regex_whitelist"; wildcard=true;;
         "-nr"| "--noreload"  ) reload=false;;
         "-d" | "--delmode"   ) addmode=false;;
         "-q" | "--quiet"     ) verbose=false;;
diff --git a/pihole b/pihole
index 411b5791..b3260d83 100755
--- a/pihole
+++ b/pihole
@@ -440,6 +440,7 @@ case "${1}" in
   "--wild" | "wildcard"         ) listFunc "$@";;
   "--regex" | "regex"           ) listFunc "$@";;
   "--whiteregex" | "whiteregex" ) listFunc "$@";;
+  "--whitewild" | "whitewild"   ) listFunc "$@";;
   "-d" | "debug"                ) debugFunc "$@";;
   "-f" | "flush"                ) flushFunc "$@";;
   "-up" | "updatePihole"        ) updatePiholeFunc "$@";;

From 6f58d58cae7ab35a523c39e268ecac2e562d16a1 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 22 Jul 2019 22:26:27 +0200
Subject: [PATCH 10/24] Add --whitewild to help texts and man pages.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 11 +++++++----
 manpages/pihole.8        | 14 ++++++++++++--
 pihole                   |  3 ++-
 3 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 0183a9e2..f0cf4701 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -32,15 +32,18 @@ helpFunc() {
     if [[ "${listType}" == "whitelist" ]]; then
         param="w"
         type="whitelist"
-    elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then
-        param="-wild"
-        type="wildcard blacklist"
     elif [[ "${listType}" == "regex_blacklist" ]]; then
         param="-regex"
         type="regex blacklist filter"
-    elif [[ "${listType}" == "regex_blacklist" ]]; then
+    elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then
+        param="-wild"
+        type="wildcard blacklist"
+    elif [[ "${listType}" == "regex_whitelist" ]]; then
         param="-whiteregex"
         type="regex whitelist filter"
+    elif [[ "${listType}" == "regex_whitelist" && "${wildcard}" == true ]]; then
+        param="-whitewild"
+        type="wildcard whitelist"
     else
         param="b"
         type="blacklist"
diff --git a/manpages/pihole.8 b/manpages/pihole.8
index 065280c7..11923392 100644
--- a/manpages/pihole.8
+++ b/manpages/pihole.8
@@ -66,14 +66,24 @@ Available commands and options:
     Adds or removes specified domain or domains to the blacklist
 .br
 
+\fB--regex, regex\fR [options] [<regex1> <regex2 ...>]
+.br
+    Add or removes specified regex filter to the regex blacklist
+.br
+
+\fB--whiteregex\fR [options] [<regex1> <regex2 ...>]
+.br
+    Add or removes specified regex filter to the regex whitelist
+.br
+
 \fB--wild, wildcard\fR [options] [<domain1> <domain2 ...>]
 .br
     Add or removes specified domain to the wildcard blacklist
 .br
 
-\fB--regex, regex\fR [options] [<regex1> <regex2 ...>]
+\fB--whitewild\fR [options] [<domain1> <domain2 ...>]
 .br
-    Add or removes specified regex filter to the regex blacklist
+    Add or removes specified domain to the wildcard whitelist
 .br
 
     (Whitelist/Blacklist manipulation options):
diff --git a/pihole b/pihole
index b3260d83..5d373a47 100755
--- a/pihole
+++ b/pihole
@@ -375,9 +375,10 @@ Add '-h' after specific commands for more information on usage
 Whitelist/Blacklist Options:
   -w, whitelist       Whitelist domain(s)
   -b, blacklist       Blacklist domain(s)
-  --wild, wildcard     Wildcard blacklist domain(s)
   --regex, regex       Regex blacklist domains(s)
   --whiteregex         Regex whitelist domains(s)
+  --wild, wildcard     Wildcard blacklist domain(s)
+  --whitewild          Wildcard whitelist domain(s)
                         Add '-h' for more info on whitelist/blacklist usage
 
 Debugging Options:

From a95b4734170c8a401a698cd03665546f3269396e Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 5 Aug 2019 20:56:01 +0200
Subject: [PATCH 11/24] Rearranage if statements to ensure the proper output is
 shown for wildcard-style filters.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index f0cf4701..31822c9a 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -32,18 +32,18 @@ helpFunc() {
     if [[ "${listType}" == "whitelist" ]]; then
         param="w"
         type="whitelist"
-    elif [[ "${listType}" == "regex_blacklist" ]]; then
-        param="-regex"
-        type="regex blacklist filter"
     elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then
         param="-wild"
         type="wildcard blacklist"
-    elif [[ "${listType}" == "regex_whitelist" ]]; then
-        param="-whiteregex"
-        type="regex whitelist filter"
+    elif [[ "${listType}" == "regex_blacklist" ]]; then
+        param="-regex"
+        type="regex blacklist filter"
     elif [[ "${listType}" == "regex_whitelist" && "${wildcard}" == true ]]; then
         param="-whitewild"
         type="wildcard whitelist"
+    elif [[ "${listType}" == "regex_whitelist" ]]; then
+        param="-whiteregex"
+        type="regex whitelist filter"
     else
         param="b"
         type="blacklist"

From 09190c1735eb7482537257cab54ea4103d8ffd18 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 5 Aug 2019 21:03:47 +0200
Subject: [PATCH 12/24] Only check once for if this is a regex list or not.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 31822c9a..60b820f5 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -94,15 +94,19 @@ HandleOther() {
 }
 
 ProcessDomainList() {
+    local is_regexlist
     if [[ "${listType}" == "regex_blacklist" ]]; then
         # Regex black filter list
         listname="regex blacklist filters"
+        is_regexlist=true
     elif [[ "${listType}" == "regex_whitelist" ]]; then
         # Regex white filter list
         listname="regex whitelist filters"
+        is_regexlist=true
     else
         # Whitelist / Blacklist
         listname="${listType}"
+        is_regexlist=false
     fi
 
     for dom in "${domList[@]}"; do
@@ -115,7 +119,7 @@ ProcessDomainList() {
         # if delmode then remove from desired list but do not add to the other
         if ${addmode}; then
             AddDomain "${dom}" "${listType}"
-            if [[ ! "${listType}" == "regex_"*"list" ]]; then
+            if ! ${is_regexlist}; then
                 RemoveDomain "${dom}" "${listAlt}"
             fi
         else

From 06860ed5b49a4554026ca4f67aa1dc8f9eed06c7 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 5 Aug 2019 21:07:39 +0200
Subject: [PATCH 13/24] Group tables have only two columns.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/piholeDebug.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index b2533eb0..e7e6b791 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1082,21 +1082,21 @@ show_groups() {
 
 show_adlists() {
     show_db_entries "Adlists" "SELECT * FROM adlist" "4 100 7 10 13 50"
-    show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 100 7 10 13 50"
+    show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 4"
 }
 
 show_whitelist() {
     show_db_entries "Exact whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50"
     show_db_entries "Regex whitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50"
-    show_db_entries "Exact whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 100 7 10 13 50"
-    show_db_entries "Regex whitelist groups" "SELECT * FROM whitelist_by_group" "4 100 7 10 13 50"
+    show_db_entries "Exact whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 4"
+    show_db_entries "Regex whitelist groups" "SELECT * FROM whitelist_by_group" "4 4"
 }
 
 show_blacklist() {
     show_db_entries "Exact blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50"
     show_db_entries "Regex blacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50"
-    show_db_entries "Exact blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 100 7 10 13 50"
-    show_db_entries "Regex blacklist groups" "SELECT * FROM blacklist_by_group" "4 100 7 10 13 50"
+    show_db_entries "Exact blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 4"
+    show_db_entries "Regex blacklist groups" "SELECT * FROM blacklist_by_group" "4 4"
 }
 
 analyze_gravity_list() {

From af754e3fc4c394276aadeec26f1068ef9ae18c2d Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 5 Aug 2019 21:08:36 +0200
Subject: [PATCH 14/24] Rearrange group tables directly next to the tables they
 refer to.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/piholeDebug.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index e7e6b791..dc353ff0 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1087,15 +1087,15 @@ show_adlists() {
 
 show_whitelist() {
     show_db_entries "Exact whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50"
-    show_db_entries "Regex whitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50"
     show_db_entries "Exact whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 4"
+    show_db_entries "Regex whitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50"
     show_db_entries "Regex whitelist groups" "SELECT * FROM whitelist_by_group" "4 4"
 }
 
 show_blacklist() {
     show_db_entries "Exact blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50"
-    show_db_entries "Regex blacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50"
     show_db_entries "Exact blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 4"
+    show_db_entries "Regex blacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50"
     show_db_entries "Regex blacklist groups" "SELECT * FROM blacklist_by_group" "4 4"
 }
 

From 6e2e825a5fe3c4148347067e58309d55e4771bf8 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 5 Aug 2019 21:10:52 +0200
Subject: [PATCH 15/24] Rename options "pihole --whiteregex" to "pihole
 --white-regex" for the sake of readability. The same applied for "whitewild"
 -> "white-wild"

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 8 ++++----
 pihole                   | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 60b820f5..6a606665 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -39,10 +39,10 @@ helpFunc() {
         param="-regex"
         type="regex blacklist filter"
     elif [[ "${listType}" == "regex_whitelist" && "${wildcard}" == true ]]; then
-        param="-whitewild"
+        param="-white-wild"
         type="wildcard whitelist"
     elif [[ "${listType}" == "regex_whitelist" ]]; then
-        param="-whiteregex"
+        param="-white-regex"
         type="regex whitelist filter"
     else
         param="b"
@@ -230,8 +230,8 @@ for var in "$@"; do
         "-b" | "blacklist"   ) listType="blacklist"; listAlt="whitelist";;
         "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;;
         "--regex" | "regex"   ) listType="regex_blacklist";;
-        "--whiteregex" | "whiteregex" ) listType="regex_whitelist";;
-        "--whitewild" | "whitewild" ) listType="regex_whitelist"; wildcard=true;;
+        "--white-regex" | "white-regex" ) listType="regex_whitelist";;
+        "--white-wild" | "white-wild" ) listType="regex_whitelist"; wildcard=true;;
         "-nr"| "--noreload"  ) reload=false;;
         "-d" | "--delmode"   ) addmode=false;;
         "-q" | "--quiet"     ) verbose=false;;
diff --git a/pihole b/pihole
index 5d373a47..dfd6eda9 100755
--- a/pihole
+++ b/pihole
@@ -440,8 +440,8 @@ case "${1}" in
   "-b" | "blacklist"            ) listFunc "$@";;
   "--wild" | "wildcard"         ) listFunc "$@";;
   "--regex" | "regex"           ) listFunc "$@";;
-  "--whiteregex" | "whiteregex" ) listFunc "$@";;
-  "--whitewild" | "whitewild"   ) listFunc "$@";;
+  "--white-regex" | "white-regex" ) listFunc "$@";;
+  "--white-wild" | "white-wild"   ) listFunc "$@";;
   "-d" | "debug"                ) debugFunc "$@";;
   "-f" | "flush"                ) flushFunc "$@";;
   "-up" | "updatePihole"        ) updatePiholeFunc "$@";;

From 4371c9ba03e48daa2f40e1dbe31c1b3bd72d2447 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 5 Aug 2019 21:20:07 +0200
Subject: [PATCH 16/24] Ensure proper permissions are set for gravity.db after
 creation.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 automated install/basic-install.sh | 3 +++
 gravity.sh                         | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh
index 1e87b943..7dbdc596 100755
--- a/automated install/basic-install.sh	
+++ b/automated install/basic-install.sh	
@@ -1908,6 +1908,9 @@ installPihole() {
             chmod a+rx /var/www/html
             # Give pihole access to the Web server group
             usermod -a -G ${LIGHTTPD_GROUP} pihole
+            # Give lighttpd access to the pihole group so the web interface can
+            # manage the gravity.db database
+            usermod -a -G pihole ${LIGHTTPD_USER}
             # If the lighttpd command is executable,
             if is_command lighty-enable-mod ; then
                 # enable fastcgi and fastcgi-php
diff --git a/gravity.sh b/gravity.sh
index f9ecc1d1..1ad43df1 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -85,6 +85,10 @@ fi
 # Generate new sqlite3 file from schema template
 generate_gravity_database() {
   sqlite3 "${gravityDBfile}" < "${gravityDBschema}"
+
+  # Ensure proper permissions are set for the newly created database
+  chown pihole:pihole "${gravityDBfile}"
+  chmod g+w "${piholeDir}" "${gravityDBfile}"
 }
 
 # Import domains from file and store them in the specified database table

From dc93462d42e77b488883f6a392e67a5561d303c5 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Tue, 6 Aug 2019 20:28:00 +0200
Subject: [PATCH 17/24] Group table has only two columns

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/piholeDebug.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index dc353ff0..38861849 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1077,7 +1077,7 @@ show_db_entries() {
 }
 
 show_groups() {
-    show_db_entries "Groups" "SELECT * FROM \"group\"" "4 100 7 10 13 50"
+    show_db_entries "Groups" "SELECT * FROM \"group\"" "4 4"
 }
 
 show_adlists() {

From 3e78ed95d4cc6a7ef50d0bca11fcc17edb8eb3c5 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 17 Aug 2019 15:04:04 +0200
Subject: [PATCH 18/24] Fix displaying options for table "group" in the
 debugger.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/piholeDebug.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index 38861849..134b15ce 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1077,7 +1077,7 @@ show_db_entries() {
 }
 
 show_groups() {
-    show_db_entries "Groups" "SELECT * FROM \"group\"" "4 4"
+    show_db_entries "Groups" "SELECT * FROM \"group\"" "4 4 30 50"
 }
 
 show_adlists() {

From b1838512b25bf3715d629c5f43fc27539f70d4ac Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 22 Aug 2019 13:39:58 +0200
Subject: [PATCH 19/24] Explicitly select columns (and their order) when
 listing the databaes tables. Print timestamps translated to strings instead
 of printing the integer timestamps.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/piholeDebug.sh | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index 134b15ce..7ba03a37 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1081,22 +1081,22 @@ show_groups() {
 }
 
 show_adlists() {
-    show_db_entries "Adlists" "SELECT * FROM adlist" "4 100 7 10 13 50"
+    show_db_entries "Adlists" "SELECT id,address,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM adlist" "4 100 7 19 19 50"
     show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 4"
 }
 
 show_whitelist() {
-    show_db_entries "Exact whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50"
-    show_db_entries "Exact whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 4"
-    show_db_entries "Regex whitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50"
-    show_db_entries "Regex whitelist groups" "SELECT * FROM whitelist_by_group" "4 4"
+    show_db_entries "Exact whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM whitelist" "4 100 7 19 19 50"
+    show_db_entries "Exact whitelist groups" "SELECT * FROM whitelist_by_group" "4 4"
+    show_db_entries "Regex whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM regex_whitelist" "4 100 7 19 19 50"
+    show_db_entries "Regex whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 4"
 }
 
 show_blacklist() {
-    show_db_entries "Exact blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50"
-    show_db_entries "Exact blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 4"
-    show_db_entries "Regex blacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50"
-    show_db_entries "Regex blacklist groups" "SELECT * FROM blacklist_by_group" "4 4"
+    show_db_entries "Exact blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM blacklist" "4 100 7 19 19 50"
+    show_db_entries "Exact blacklist groups" "SELECT * FROM blacklist_by_group" "4 4"
+    show_db_entries "Regex blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM regex_blacklist" "4 100 7 19 19 50"
+    show_db_entries "Regex blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 4"
 }
 
 analyze_gravity_list() {

From cc40c18f49ccf9dcf926cd18bb9dc1880345be3c Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 22 Aug 2019 13:54:46 +0200
Subject: [PATCH 20/24] Wrap upgrade script commands in a transaction.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/database_migration/gravity/1_to_2.sql | 6 ++++++
 advanced/Scripts/database_migration/gravity/2_to_3.sql | 4 ++++
 2 files changed, 10 insertions(+)

diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql
index 90a48418..6d57a6fe 100644
--- a/advanced/Scripts/database_migration/gravity/1_to_2.sql
+++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql
@@ -1,3 +1,7 @@
+.timeout 30000
+
+BEGIN TRANSACTION;
+
 CREATE TABLE domain_audit
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -6,3 +10,5 @@ CREATE TABLE domain_audit
 );
 
 UPDATE info SET value = 2 WHERE property = 'version';
+
+COMMIT;
diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql
index e121f28c..fd7c24d2 100644
--- a/advanced/Scripts/database_migration/gravity/2_to_3.sql
+++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql
@@ -2,6 +2,8 @@
 
 PRAGMA FOREIGN_KEYS=OFF;
 
+BEGIN TRANSACTION;
+
 ALTER TABLE regex RENAME TO regex_blacklist;
 
 CREATE TABLE regex_blacklist_by_group
@@ -59,3 +61,5 @@ CREATE TRIGGER tr_regex_whitelist_update AFTER UPDATE ON regex_whitelist
 
 
 UPDATE info SET value = 3 WHERE property = 'version';
+
+COMMIT;

From aef7892de68a272df0dd62bd18e28a8790af66fa Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 22 Aug 2019 13:57:01 +0200
Subject: [PATCH 21/24] Add missing hyphens.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 manpages/pihole.8 | 4 ++--
 pihole            | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/manpages/pihole.8 b/manpages/pihole.8
index 11923392..ed012092 100644
--- a/manpages/pihole.8
+++ b/manpages/pihole.8
@@ -71,7 +71,7 @@ Available commands and options:
     Add or removes specified regex filter to the regex blacklist
 .br
 
-\fB--whiteregex\fR [options] [<regex1> <regex2 ...>]
+\fB--white-regex\fR [options] [<regex1> <regex2 ...>]
 .br
     Add or removes specified regex filter to the regex whitelist
 .br
@@ -81,7 +81,7 @@ Available commands and options:
     Add or removes specified domain to the wildcard blacklist
 .br
 
-\fB--whitewild\fR [options] [<domain1> <domain2 ...>]
+\fB--white-wild\fR [options] [<domain1> <domain2 ...>]
 .br
     Add or removes specified domain to the wildcard whitelist
 .br
diff --git a/pihole b/pihole
index dfd6eda9..1d9f0809 100755
--- a/pihole
+++ b/pihole
@@ -375,10 +375,10 @@ Add '-h' after specific commands for more information on usage
 Whitelist/Blacklist Options:
   -w, whitelist       Whitelist domain(s)
   -b, blacklist       Blacklist domain(s)
-  --regex, regex       Regex blacklist domains(s)
-  --whiteregex         Regex whitelist domains(s)
-  --wild, wildcard     Wildcard blacklist domain(s)
-  --whitewild          Wildcard whitelist domain(s)
+  --regex, regex      Regex blacklist domains(s)
+  --white-regex       Regex whitelist domains(s)
+  --wild, wildcard    Wildcard blacklist domain(s)
+  --white-wild        Wildcard whitelist domain(s)
                         Add '-h' for more info on whitelist/blacklist usage
 
 Debugging Options:

From 42ccc1ef24cf80d53d0b20e9e01d0a6d1a66a992 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 22 Aug 2019 14:06:42 +0200
Subject: [PATCH 22/24] Add support for regex whitelist in "pihole -q".

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/query.sh | 28 ++++++++++++++++++----------
 1 file changed, 18 insertions(+), 10 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index a587f238..4bffe251 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -128,25 +128,24 @@ scanDatabaseTable() {
     done
 }
 
-# Scan Whitelist and Blacklist
-scanDatabaseTable "${domainQuery}" "whitelist" "${exact}"
-scanDatabaseTable "${domainQuery}" "blacklist" "${exact}"
-
-# Scan Regex table
-mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_blacklist" 2> /dev/null)
+scanRegexDatabaseTable() {
+local domain list
+domain="${1}"
+list="${2}"
+mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_${list}" 2> /dev/null)
 
 # If we have regexps to process
 if [[ "${#regexList[@]}" -ne 0 ]]; then
 	# Split regexps over a new line
 	str_regexList=$(printf '%s\n' "${regexList[@]}")
-	# Check domainQuery against regexps
-	mapfile -t regexMatches < <(scanList "${domainQuery}" "${str_regexList}" "regex")
+	# Check domain against regexps
+	mapfile -t regexMatches < <(scanList "${domain}" "${str_regexList}" "regex")
 	# If there were regex matches
 	if [[  "${#regexMatches[@]}" -ne 0 ]]; then
 		# Split matching regexps over a new line
 		str_regexMatches=$(printf '%s\n' "${regexMatches[@]}")
 		# Form a "matched" message
-		str_message="${matchType^} found in ${COL_BOLD}Regex list${COL_NC}"
+		str_message="${matchType^} found in ${COL_BOLD}Regex ${list}${COL_NC}"
 		# Form a "results" message
 		str_result="${COL_BOLD}${str_regexMatches}${COL_NC}"
 		# If we are displaying more than just the source of the block
@@ -159,11 +158,20 @@ if [[ "${#regexList[@]}" -ne 0 ]]; then
 			# shellcheck disable=SC2001
 			echo "${str_result}" | sed 's/^/   /'
 		else
-			echo "π Regex list"
+			echo "π Regex ${list}"
 			exit 0
 		fi
 	fi
 fi
+}
+
+# Scan Whitelist and Blacklist
+scanDatabaseTable "${domainQuery}" "whitelist" "${exact}"
+scanDatabaseTable "${domainQuery}" "blacklist" "${exact}"
+
+# Scan Regex table
+scanRegexDatabaseTable "${domainQuery}" "whitelist"
+scanRegexDatabaseTable "${domainQuery}" "blacklist"
 
 # Get version sorted *.domains filenames (without dir path)
 lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)")

From 23b688287f4fdb9c809053880594f894c4cfbfc8 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 22 Aug 2019 14:12:58 +0200
Subject: [PATCH 23/24] Fix indentation in query.sh. No functional change in
 this commit.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/query.sh | 88 ++++++++++++++++++++-------------------
 1 file changed, 45 insertions(+), 43 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 4bffe251..035adaac 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -36,14 +36,14 @@ scanList(){
     # /dev/null forces filename to be printed when only one list has been generated
     # shellcheck disable=SC2086
     case "${type}" in
-		"exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
-		# Create array of regexps
-		# Iterate through each regexp and check whether it matches the domainQuery
-		# If it does, print the matching regexp and continue looping
-		# Input 1 - regexps | Input 2 - domainQuery
-		"regex" ) awk 'NR==FNR{regexps[$0];next}{for (r in regexps)if($0 ~ r)print r}' \
-					<(echo "${lists}") <(echo "${domain}") 2>/dev/null;;
-		*       ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;;
+        "exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
+        # Create array of regexps
+        # Iterate through each regexp and check whether it matches the domainQuery
+        # If it does, print the matching regexp and continue looping
+        # Input 1 - regexps | Input 2 - domainQuery
+        "regex" ) awk 'NR==FNR{regexps[$0];next}{for (r in regexps)if($0 ~ r)print r}' \
+                  <(echo "${lists}") <(echo "${domain}") 2>/dev/null;;
+        *       ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;;
     esac
 }
 
@@ -100,8 +100,8 @@ scanDatabaseTable() {
     # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
     # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
     case "${type}" in
-		"exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
-		*       ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
+        "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
+        *       ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
     esac
 
     # Send prepared query to gravity database
@@ -129,40 +129,42 @@ scanDatabaseTable() {
 }
 
 scanRegexDatabaseTable() {
-local domain list
-domain="${1}"
-list="${2}"
-mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_${list}" 2> /dev/null)
+    local domain list
+    domain="${1}"
+    list="${2}"
 
-# If we have regexps to process
-if [[ "${#regexList[@]}" -ne 0 ]]; then
-	# Split regexps over a new line
-	str_regexList=$(printf '%s\n' "${regexList[@]}")
-	# Check domain against regexps
-	mapfile -t regexMatches < <(scanList "${domain}" "${str_regexList}" "regex")
-	# If there were regex matches
-	if [[  "${#regexMatches[@]}" -ne 0 ]]; then
-		# Split matching regexps over a new line
-		str_regexMatches=$(printf '%s\n' "${regexMatches[@]}")
-		# Form a "matched" message
-		str_message="${matchType^} found in ${COL_BOLD}Regex ${list}${COL_NC}"
-		# Form a "results" message
-		str_result="${COL_BOLD}${str_regexMatches}${COL_NC}"
-		# If we are displaying more than just the source of the block
-		if [[ -z "${blockpage}" ]]; then
-			# Set the wildcard match flag
-			wcMatch=true
-			# Echo the "matched" message, indented by one space
-			echo " ${str_message}"
-			# Echo the "results" message, each line indented by three spaces
-			# shellcheck disable=SC2001
-			echo "${str_result}" | sed 's/^/   /'
-		else
-			echo "π Regex ${list}"
-			exit 0
-		fi
-	fi
-fi
+    # Query all regex from the corresponding database tables
+    mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_${list}" 2> /dev/null)
+
+    # If we have regexps to process
+    if [[ "${#regexList[@]}" -ne 0 ]]; then
+        # Split regexps over a new line
+        str_regexList=$(printf '%s\n' "${regexList[@]}")
+        # Check domain against regexps
+        mapfile -t regexMatches < <(scanList "${domain}" "${str_regexList}" "regex")
+        # If there were regex matches
+        if [[  "${#regexMatches[@]}" -ne 0 ]]; then
+            # Split matching regexps over a new line
+            str_regexMatches=$(printf '%s\n' "${regexMatches[@]}")
+            # Form a "matched" message
+            str_message="${matchType^} found in ${COL_BOLD}Regex ${list}${COL_NC}"
+            # Form a "results" message
+            str_result="${COL_BOLD}${str_regexMatches}${COL_NC}"
+            # If we are displaying more than just the source of the block
+            if [[ -z "${blockpage}" ]]; then
+                # Set the wildcard match flag
+                wcMatch=true
+                # Echo the "matched" message, indented by one space
+                echo " ${str_message}"
+                # Echo the "results" message, each line indented by three spaces
+                # shellcheck disable=SC2001
+                echo "${str_result}" | sed 's/^/   /'
+            else
+                echo "π Regex ${list}"
+                exit 0
+            fi
+        fi
+    fi
 }
 
 # Scan Whitelist and Blacklist

From 6faddfcd3d0a0ae00c0de3833b43013c60efec96 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 23 Aug 2019 10:09:52 +0200
Subject: [PATCH 24/24] Print timestamps in local time zone of the Pi-hole.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/piholeDebug.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index c9280f45..84e34416 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -1109,21 +1109,21 @@ show_groups() {
 }
 
 show_adlists() {
-    show_db_entries "Adlists" "SELECT id,address,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM adlist" "4 100 7 19 19 50"
+    show_db_entries "Adlists" "SELECT id,address,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM adlist" "4 100 7 19 19 50"
     show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 4"
 }
 
 show_whitelist() {
-    show_db_entries "Exact whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM whitelist" "4 100 7 19 19 50"
+    show_db_entries "Exact whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM whitelist" "4 100 7 19 19 50"
     show_db_entries "Exact whitelist groups" "SELECT * FROM whitelist_by_group" "4 4"
-    show_db_entries "Regex whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM regex_whitelist" "4 100 7 19 19 50"
+    show_db_entries "Regex whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM regex_whitelist" "4 100 7 19 19 50"
     show_db_entries "Regex whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 4"
 }
 
 show_blacklist() {
-    show_db_entries "Exact blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM blacklist" "4 100 7 19 19 50"
+    show_db_entries "Exact blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM blacklist" "4 100 7 19 19 50"
     show_db_entries "Exact blacklist groups" "SELECT * FROM blacklist_by_group" "4 4"
-    show_db_entries "Regex blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM regex_blacklist" "4 100 7 19 19 50"
+    show_db_entries "Regex blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM regex_blacklist" "4 100 7 19 19 50"
     show_db_entries "Regex blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 4"
 }