From 4fd412d7c3976c2ce87b899c76db82427c85c7ef Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 13 Jul 2020 20:47:12 +0200 Subject: [PATCH 1/2] Add date_updated field in adlist table set when a list changes. Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 6 ++++++ .../Scripts/database_migration/gravity/12_to_13.sql | 11 +++++++++++ advanced/Templates/gravity.db.sql | 5 +++-- gravity.sh | 13 +++++++++++++ 4 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/12_to_13.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 70090a3b..e2740a7f 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -110,4 +110,10 @@ upgrade_gravityDB(){ sqlite3 "${database}" < "${scriptPath}/11_to_12.sql" version=12 fi + if [[ "$version" == "12" ]]; then + # Add column date_updated to alist table + echo -e " ${INFO} Upgrading gravity database from version 12 to 13" + sqlite3 "${database}" < "${scriptPath}/12_to_13.sql" + version=12 + fi } diff --git a/advanced/Scripts/database_migration/gravity/12_to_13.sql b/advanced/Scripts/database_migration/gravity/12_to_13.sql new file mode 100644 index 00000000..48dad69f --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/12_to_13.sql @@ -0,0 +1,11 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +ALTER TABLE adlist ADD COLUMN date_updated INTEGER; + +UPDATE info SET value = 13 WHERE property = 'version'; + +COMMIT; \ No newline at end of file diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index fcf3489b..e6cb4c59 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -31,7 +31,8 @@ CREATE TABLE adlist enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + comment TEXT, + date_updated INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) ); CREATE TABLE adlist_by_group @@ -53,7 +54,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','12'); +INSERT INTO "info" VALUES('version','13'); CREATE TABLE domain_audit ( diff --git a/gravity.sh b/gravity.sh index ca9363e7..c5d8dca2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -207,6 +207,17 @@ database_table_from_file() { echo -e " ${CROSS} Unable to remove ${tmpFile}" } +# Update timestamp of last update of this list. We store this in the "old" database as all values in the new database will later be overwritten +database_adlist_updated() { + output=$( { printf ".timeout 30000\\nUPDATE adlist SET date_updated = (cast(strftime('%%s', 'now') as int)) WHERE id = %i;\\n" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to update timestamp of adlist with ID ${1} in database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi +} + # Migrate pre-v5.0 list files to database-based Pi-hole versions migrate_to_database() { # Create database file only if not present @@ -555,6 +566,8 @@ gravity_DownloadBlocklistFromUrl() { gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" # Add domains to database table file parseList "${adlistID}" "${saveLocation}" "${target}" + # Update date_updated field in gravity database table + database_adlist_updated "${adlistID}" else # Fall back to previously cached list if $patternBuffer is empty echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" From 12b9748c43bc7178e9cbef35806267221e5b4659 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 7 Sep 2020 21:11:57 +0200 Subject: [PATCH 2/2] Fine-tune adlist:adte_modified trigger and don't default new date_updated column to now but use NULL instead Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity/12_to_13.sql | 7 +++++++ advanced/Templates/gravity.db.sql | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity/12_to_13.sql b/advanced/Scripts/database_migration/gravity/12_to_13.sql index 48dad69f..d16791d6 100644 --- a/advanced/Scripts/database_migration/gravity/12_to_13.sql +++ b/advanced/Scripts/database_migration/gravity/12_to_13.sql @@ -6,6 +6,13 @@ BEGIN TRANSACTION; ALTER TABLE adlist ADD COLUMN date_updated INTEGER; +DROP TRIGGER tr_adlist_update; + +CREATE TRIGGER tr_adlist_update AFTER UPDATE OF address,enabled,comment ON adlist + BEGIN + UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE id = NEW.id; + END; + UPDATE info SET value = 13 WHERE property = 'version'; COMMIT; \ No newline at end of file diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index e6cb4c59..2aa7e8f8 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -32,7 +32,7 @@ CREATE TABLE adlist date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT, - date_updated INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) + date_updated INTEGER ); CREATE TABLE adlist_by_group @@ -86,9 +86,9 @@ CREATE TABLE client_by_group PRIMARY KEY (client_id, group_id) ); -CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist +CREATE TRIGGER tr_adlist_update AFTER UPDATE OF address,enabled,comment ON adlist BEGIN - UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; + UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE id = NEW.id; END; CREATE TRIGGER tr_client_update AFTER UPDATE ON client