From 00cbb8bc8abb119ebc3c3f0d912b94a334e06128 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 5 Jul 2023 22:24:11 +0200 Subject: [PATCH 1/6] Add antigravity support to gravity Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 7 +++++ .../database_migration/gravity/16_to_17.sql | 13 ++++++++++ advanced/Templates/gravity.db.sql | 16 +++++++++++- gravity.sh | 26 +++++++++++++------ 4 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/16_to_17.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 851ddb0a..e36d9b1e 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -134,4 +134,11 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 "${database}" < "${scriptPath}/15_to_16.sql" version=16 fi + if [[ "$version" == "16" ]]; then + # Add antigravity table + # Add column type to adlist table (to support adlist types) + echo -e " ${INFO} Upgrading gravity database from version 16 to 17" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/16_to_17.sql" + version=17 + fi } diff --git a/advanced/Scripts/database_migration/gravity/16_to_17.sql b/advanced/Scripts/database_migration/gravity/16_to_17.sql new file mode 100644 index 00000000..c7b9049b --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/16_to_17.sql @@ -0,0 +1,13 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +ALTER TABLE adlist ADD COLUMN type INTEGER NOT NULL DEFAULT 0; + +UPDATE adlist SET type = 0; + +UPDATE info SET value = 17 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 881cfcc3..17712cf7 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -36,7 +36,8 @@ CREATE TABLE adlist number INTEGER NOT NULL DEFAULT 0, invalid_domains INTEGER NOT NULL DEFAULT 0, status INTEGER NOT NULL DEFAULT 0, - abp_entries INTEGER NOT NULL DEFAULT 0 + abp_entries INTEGER NOT NULL DEFAULT 0, + type INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE adlist_by_group @@ -52,6 +53,12 @@ CREATE TABLE gravity adlist_id INTEGER NOT NULL REFERENCES adlist (id) ); +CREATE TABLE antigravity +( + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id) +); + CREATE TABLE info ( property TEXT PRIMARY KEY, @@ -144,6 +151,13 @@ CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); +CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + CREATE VIEW vw_adlist AS SELECT DISTINCT address, id FROM adlist WHERE enabled = 1 diff --git a/gravity.sh b/gravity.sh index ed402a34..d784a2e7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -361,6 +361,7 @@ gravity_DownloadBlocklists() { # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceTypes <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( @@ -382,7 +383,7 @@ gravity_DownloadBlocklists() { unset sources fi - local url domain agent str target compression + local url domain agent str target compression adlist_type echo "" # Prepare new gravity database @@ -394,7 +395,7 @@ gravity_DownloadBlocklists() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to create new database ${gravityTEMPfile}\\n ${output}" - gravity_Cleanup "error" + #gravity_Cleanup "error" else echo -e "${OVER} ${TICK} ${str}" fi @@ -433,6 +434,15 @@ gravity_DownloadBlocklists() { url="${sources[$i]}" domain="${sourceDomains[$i]}" id="${sourceIDs[$i]}" + if [[ "${sourceTypes[$i]}" -eq "0" ]]; then + # Gravity list + str="blocklist" + adlist_type="gravity" + else + # AntiGravity list + str="allowlist" + adlist_type="antigravity" + fi # Save the file as list.#.domain saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}" @@ -441,7 +451,7 @@ gravity_DownloadBlocklists() { # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-) agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" - echo -e " ${INFO} Target: ${url}" + echo -e " ${INFO} Target: ${url} (${str})" local regex check_url # Check for characters NOT allowed in URLs regex="[^a-zA-Z0-9:/?&%=~._()-;]" @@ -453,7 +463,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" + gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" fi echo "" done @@ -485,7 +495,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" + local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" gravity_type="${7}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM @@ -579,7 +589,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table file - pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "2" done="true" # Check if $listCurlBuffer is a non-zero length file @@ -589,7 +599,7 @@ gravity_DownloadBlocklistFromUrl() { # Remove curl buffer file after its use rm "${listCurlBuffer}" # Add domains to database table file - pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" # Compare lists, are they identical? compareLists "${adlistID}" "${saveLocation}" done="true" @@ -605,7 +615,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" # Add domains to database table file - pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" From 35512c4dc9071502b10f251d514d297a79fe7ea5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jul 2023 19:19:55 +0200 Subject: [PATCH 2/6] Fix adlist.list migration step failing during tests Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index d784a2e7..89b4ab84 100755 --- a/gravity.sh +++ b/gravity.sh @@ -178,7 +178,7 @@ database_table_from_file() { echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" elif [[ "${table}" == "adlist" ]]; then # Adlist table format - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0,0" >> "${tmpFile}" else # White-, black-, and regexlist table format echo "${rowid},${list_type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\"" >> "${tmpFile}" From 375d4d9bc13b668682ded29efd2387d62d5623c5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jul 2023 21:33:59 +0200 Subject: [PATCH 3/6] Add type as new field of view vw_adlist Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 17712cf7..46f26ba7 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -65,7 +65,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','16'); +INSERT INTO "info" VALUES('version','17'); CREATE TABLE domain_audit ( @@ -158,7 +158,7 @@ CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_i LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; -CREATE VIEW vw_adlist AS SELECT DISTINCT address, id +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type FROM adlist WHERE enabled = 1 ORDER BY id; From 2a03671fb9f15cfe7e6ee378350d7f32d2e04000 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jul 2023 22:52:28 +0200 Subject: [PATCH 4/6] Reinstall gravity cleanup on error Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 89b4ab84..acf55ae3 100755 --- a/gravity.sh +++ b/gravity.sh @@ -395,7 +395,7 @@ gravity_DownloadBlocklists() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to create new database ${gravityTEMPfile}\\n ${output}" - #gravity_Cleanup "error" + gravity_Cleanup "error" else echo -e "${OVER} ${TICK} ${str}" fi From df92b8ac14988ddeece92b733dec7e8b9fb84d3a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 12 Sep 2023 20:43:34 +0200 Subject: [PATCH 5/6] Add missing creation of view vw_antigravity as well as schema change to vw_adlist Signed-off-by: DL6ER --- .../database_migration/gravity/15_to_16.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity/15_to_16.sql b/advanced/Scripts/database_migration/gravity/15_to_16.sql index c6159f40..7c3b8197 100644 --- a/advanced/Scripts/database_migration/gravity/15_to_16.sql +++ b/advanced/Scripts/database_migration/gravity/15_to_16.sql @@ -6,6 +6,20 @@ BEGIN TRANSACTION; ALTER TABLE adlist ADD COLUMN abp_entries INTEGER NOT NULL DEFAULT 0; +CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + +DROP VIEW vw_adlist; + +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type + FROM adlist + WHERE enabled = 1 + ORDER BY id; + UPDATE info SET value = 16 WHERE property = 'version'; COMMIT; From ea23c8364dd75190b1a8765e8a302877063b9b99 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Sep 2023 10:04:29 +0200 Subject: [PATCH 6/6] Move antigravity-related changed to gravits database migration step 16->17 Signed-off-by: DL6ER --- .../database_migration/gravity/15_to_16.sql | 14 -------------- .../database_migration/gravity/16_to_17.sql | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity/15_to_16.sql b/advanced/Scripts/database_migration/gravity/15_to_16.sql index 7c3b8197..c6159f40 100644 --- a/advanced/Scripts/database_migration/gravity/15_to_16.sql +++ b/advanced/Scripts/database_migration/gravity/15_to_16.sql @@ -6,20 +6,6 @@ BEGIN TRANSACTION; ALTER TABLE adlist ADD COLUMN abp_entries INTEGER NOT NULL DEFAULT 0; -CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id - FROM antigravity - LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id - LEFT JOIN adlist ON adlist.id = antigravity.adlist_id - LEFT JOIN "group" ON "group".id = adlist_by_group.group_id - WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; - -DROP VIEW vw_adlist; - -CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type - FROM adlist - WHERE enabled = 1 - ORDER BY id; - UPDATE info SET value = 16 WHERE property = 'version'; COMMIT; diff --git a/advanced/Scripts/database_migration/gravity/16_to_17.sql b/advanced/Scripts/database_migration/gravity/16_to_17.sql index c7b9049b..23532e3a 100644 --- a/advanced/Scripts/database_migration/gravity/16_to_17.sql +++ b/advanced/Scripts/database_migration/gravity/16_to_17.sql @@ -8,6 +8,20 @@ ALTER TABLE adlist ADD COLUMN type INTEGER NOT NULL DEFAULT 0; UPDATE adlist SET type = 0; +CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + +DROP VIEW vw_adlist; + +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type + FROM adlist + WHERE enabled = 1 + ORDER BY id; + UPDATE info SET value = 17 WHERE property = 'version'; COMMIT;