From 054c7a2c050b416ac7de09e29351bdb69564430d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 8 Jul 2019 21:23:46 +0200 Subject: [PATCH] Create new table + view regex_whitelist + rename old regex table to regex_blacklist. This updates the gravity.db version to 3. Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 7 +++++ .../database_migration/gravity/2_to_3.sql | 30 +++++++++++++++++++ advanced/Scripts/list.sh | 21 ++++++------- advanced/Scripts/piholeDebug.sh | 11 +++++-- advanced/Scripts/query.sh | 2 +- gravity.sh | 9 +++--- 6 files changed, 62 insertions(+), 18 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/2_to_3.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 51a3480b..f37ce176 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -29,4 +29,11 @@ upgrade_gravityDB(){ database_table_from_file "domain_audit" "${auditFile}" fi fi + if [[ "$version" == "2" ]]; then + # This migration script upgrades the gravity.db file by + # renaming the regex table to regex_blacklist, and + # creating a new regex_whitelist table + corresponding linking table and views + sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/2_to_3.sql" + version=3 + fi } diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql new file mode 100644 index 00000000..e368592a --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql @@ -0,0 +1,30 @@ +PRAGMA FOREIGN_KEYS=OFF; + +ALTER TABLE regex RENAME TO regex_blacklist; + +CREATE TABLE regex_blacklist_by_group +( + regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_blacklist_id, group_id) +); + +INSERT INTO regex_blacklist_by_group SELECT * FROM regex_by_group; +DROP TABLE regex_by_group; +DROP VIEW vw_regex; +DROP TRIGGER tr_regex_update; + +CREATE VIEW vw_regex_blacklist AS SELECT DISTINCT domain + FROM regex + LEFT JOIN regex_blacklist_by_group ON regex_blacklist_by_group.regex_blacklist_id = regex_blacklist.id + LEFT JOIN "group" ON "group".id = regex_blacklist_by_group.group_id + WHERE regex_blacklist.enabled = 1 AND (regex_blacklist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY regex_blacklist.id; + +CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist + BEGIN + UPDATE regex_blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + + +UPDATE info SET value = 3 WHERE property = 'version'; diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index fa81348b..84acf4fc 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -32,12 +32,12 @@ helpFunc() { if [[ "${listType}" == "whitelist" ]]; then param="w" type="whitelist" - elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then + elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then param="-wild" type="wildcard blacklist" - elif [[ "${listType}" == "regex" ]]; then + elif [[ "${listType}" == "regex_blacklist" ]]; then param="-regex" - type="regex filter" + type="regex blacklist filter" else param="b" type="blacklist" @@ -58,7 +58,8 @@ Options: exit 0 } -EscapeRegexp() { +Escape +Regexp() { # This way we may safely insert an arbitrary # string in our regular expressions # This sed is intentionally executed in three steps to ease maintainability @@ -72,7 +73,7 @@ HandleOther() { # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then - if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then + if [[ "${listType}" == "regex_blacklist" && "${wildcard}" == false ]]; then validDomain="${domain}" else validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check @@ -88,9 +89,9 @@ HandleOther() { } ProcessDomainList() { - if [[ "${listType}" == "regex" ]]; then + if [[ "${listType}" == "regex_blacklist" ]]; then # Regex filter list - listname="regex filters" + listname="regex blacklist filters" else # Whitelist / Blacklist listname="${listType}" @@ -106,7 +107,7 @@ ProcessDomainList() { # if delmode then remove from desired list but do not add to the other if ${addmode}; then AddDomain "${dom}" "${listType}" - if [[ ! "${listType}" == "regex" ]]; then + if [[ ! "${listType}" == "regex_blacklist" ]]; then RemoveDomain "${dom}" "${listAlt}" fi else @@ -215,8 +216,8 @@ for var in "$@"; do case "${var}" in "-w" | "whitelist" ) listType="whitelist"; listAlt="blacklist";; "-b" | "blacklist" ) listType="blacklist"; listAlt="whitelist";; - "--wild" | "wildcard" ) listType="regex"; wildcard=true;; - "--regex" | "regex" ) listType="regex";; + "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;; + "--regex" | "regex" ) listType="regex_blacklist";; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 2bfeef2d..965250f0 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1088,8 +1088,12 @@ show_blacklist() { show_db_entries "Blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50" } -show_regexlist() { - show_db_entries "Regexlist" "SELECT * FROM regex" "4 100 7 10 13 50" +show_regexblacklist() { + show_db_entries "Regexblacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50" +} + +show_regexwhitelist() { + show_db_entries "Regexwhitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50" } analyze_gravity_list() { @@ -1268,7 +1272,8 @@ analyze_gravity_list show_adlists show_whitelist show_blacklist -show_regexlist +show_regexblacklist +show_regexwhitelist show_content_of_pihole_files parse_locale analyze_pihole_log diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 4fc82744..a587f238 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -133,7 +133,7 @@ scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" # Scan Regex table -mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null) +mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_blacklist" 2> /dev/null) # If we have regexps to process if [[ "${#regexList[@]}" -ne 0 ]]; then diff --git a/gravity.sh b/gravity.sh index 89f77ce0..a379e248 100755 --- a/gravity.sh +++ b/gravity.sh @@ -183,7 +183,7 @@ migrate_to_database() { if [ -e "${regexFile}" ]; then # Store regex domains in database echo -e " ${INFO} Migrating content of ${regexFile} into new database" - database_table_from_file "regex" "${regexFile}" + database_table_from_file "regex_blacklist" "${regexFile}" fi fi @@ -591,9 +591,10 @@ gravity_Table_Count() { # Output count of blacklisted domains and regex filters gravity_ShowCount() { - gravity_Table_Count "blacklist" "blacklisted domains" - gravity_Table_Count "whitelist" "whitelisted domains" - gravity_Table_Count "regex" "regex filters" + gravity_Table_Count "blacklist" "exact blacklisted domains" + gravity_Table_Count "regex_blacklist" "regex blacklist filters" + gravity_Table_Count "whitelist" "exact whitelisted domains" + gravity_Table_Count "regex_whitelist" "regex whitelist filters" } # Parse list of domains into hosts format