Merge pull request #3951 from pi-hole/new/gravity_adlist_infos

Store gravity details in gravity.db adlist table

advanced/Scripts/database_migration/gravity-db.sh

@@ -111,9 +111,15 @@ upgrade_gravityDB(){
 		version=12
 	fi
 	if [[ "$version" == "12" ]]; then
-		# Add column date_updated to alist table
+		# Add column date_updated to adlist table
 		echo -e "  ${INFO} Upgrading gravity database from version 12 to 13"
 		sqlite3 "${database}" < "${scriptPath}/12_to_13.sql"
 		version=13
 	fi
+	if [[ "$version" == "13" ]]; then
+		# Add columns number and status to adlist table
+		echo -e "  ${INFO} Upgrading gravity database from version 13 to 14"
+		sqlite3 "${database}" < "${scriptPath}/13_to_14.sql"
+		version=14
+	fi
 }

advanced/Scripts/database_migration/gravity/13_to_14.sql

@@ -0,0 +1,13 @@
+.timeout 30000
+
+PRAGMA FOREIGN_KEYS=OFF;
+
+BEGIN TRANSACTION;
+
+ALTER TABLE adlist ADD COLUMN number INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE adlist ADD COLUMN invalid_domains INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE adlist ADD COLUMN status INTEGER NOT NULL DEFAULT 0;
+
+UPDATE info SET value = 14 WHERE property = 'version';
+
+COMMIT;

advanced/Templates/gravity.db.sql

@@ -32,7 +32,10 @@ CREATE TABLE adlist
 	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
 	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
 	comment TEXT,
-	date_updated INTEGER
+	date_updated INTEGER,
+	number INTEGER NOT NULL DEFAULT 0,
+	invalid_domains INTEGER NOT NULL DEFAULT 0,
+	status INTEGER NOT NULL DEFAULT 0
 );
 
 CREATE TABLE adlist_by_group
@@ -54,7 +57,7 @@ CREATE TABLE info
 	value TEXT NOT NULL
 );
 
-INSERT INTO "info" VALUES('version','13');
+INSERT INTO "info" VALUES('version','14');
 
 CREATE TABLE domain_audit
 (

gravity.sh

@@ -217,6 +217,48 @@ database_adlist_updated() {
   fi
 }
 
+# Check if a column with name ${2} exists in gravity table with name ${1}
+gravity_column_exists() {
+  output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
+  if [[ "${output}" == "1" ]]; then
+    return 0 # Bash 0 is success
+  fi
+
+  return 1 # Bash non-0 is failure
+}
+
+# Update number of domain on this list. We store this in the "old" database as all values in the new database will later be overwritten
+database_adlist_number() {
+  # Only try to set number of domains when this field exists in the gravity database
+  if ! gravity_column_exists "adlist" "number"; then
+    return;
+  fi
+
+  output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_lines}" "${num_invalid}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to update number of domains in adlist with ID ${1} in database ${gravityDBfile}\\n  ${output}"
+    gravity_Cleanup "error"
+  fi
+}
+
+# Update status of this list. We store this in the "old" database as all values in the new database will later be overwritten
+database_adlist_status() {
+  # Only try to set the status when this field exists in the gravity database
+  if ! gravity_column_exists "adlist" "status"; then
+    return;
+  fi
+
+  output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to update status of adlist with ID ${1} in database ${gravityDBfile}\\n  ${output}"
+    gravity_Cleanup "error"
+  fi
+}
+
 # Migrate pre-v5.0 list files to database-based Pi-hole versions
 migrate_to_database() {
   # Create database file only if not present
@@ -444,6 +486,8 @@ gravity_DownloadBlocklists() {
 }
 
 total_num=0
+num_lines=0
+num_invalid=0
 parseList() {
   local adlistID="${1}" src="${2}" target="${3}" incorrect_lines
   # This sed does the following things:
@@ -454,7 +498,7 @@ parseList() {
   # Find (up to) five domains containing invalid characters (see above)
   incorrect_lines="$(sed -e "/[^a-zA-Z0-9.\_-]/!d" "${src}" | head -n 5)"
 
-  local num_lines num_target_lines num_correct_lines num_invalid
+  local num_target_lines num_correct_lines num_invalid
   # Get number of lines in source file
   num_lines="$(grep -c "^" "${src}")"
   # Get number of lines in destination file
@@ -463,9 +507,9 @@ parseList() {
   total_num="$num_target_lines"
   num_invalid="$(( num_lines-num_correct_lines ))"
   if [[ "${num_invalid}" -eq 0 ]]; then
-    echo "  ${INFO} Received ${num_lines} domains"
+    echo "  ${INFO} Analyzed ${num_lines} domains"
   else
-    echo "  ${INFO} Received ${num_lines} domains, ${num_invalid} domains invalid!"
+    echo "  ${INFO} Analyzed ${num_lines} domains, ${num_invalid} domains invalid!"
   fi
 
   # Display sample of invalid lines if we found some
@@ -476,6 +520,29 @@ parseList() {
     done <<< "${incorrect_lines}"
   fi
 }
+compareLists() {
+  local adlistID="${1}" target="${2}"
+
+  # Verify checksum when an older checksum exists
+  if [[ -s "${target}.sha1" ]]; then
+    if ! sha1sum --check --status --strict "${target}.sha1"; then
+      # The list changed upstream, we need to update the checksum
+      sha1sum "${target}" > "${target}.sha1"
+      echo "  ${INFO} List has been updated"
+      database_adlist_status "${adlistID}" "1"
+      database_adlist_updated "${adlistID}"
+    else
+      echo "  ${INFO} List stayed unchanged"
+      database_adlist_status "${adlistID}" "2"
+    fi
+  else
+    # No checksum available, create one for comparing on the next run
+    sha1sum "${target}" > "${target}.sha1"
+    # We assume here it was changed upstream
+    database_adlist_status "${adlistID}" "1"
+    database_adlist_updated "${adlistID}"
+  fi
+}
 
 # Download specified URL and perform checks on HTTP status and file content
 gravity_DownloadBlocklistFromUrl() {
@@ -559,31 +626,49 @@ gravity_DownloadBlocklistFromUrl() {
       esac;;
   esac
 
+  local done="false"
   # Determine if the blocklist was downloaded and saved correctly
   if [[ "${success}" == true ]]; then
     if [[ "${httpCode}" == "304" ]]; then
       # Add domains to database table file
       parseList "${adlistID}" "${saveLocation}" "${target}"
+      database_adlist_status "${adlistID}" "2"
+      database_adlist_number "${adlistID}"
+      done="true"
     # Check if $patternbuffer is a non-zero length file
     elif [[ -s "${patternBuffer}" ]]; then
       # Determine if blocklist is non-standard and parse as appropriate
       gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}"
       # Add domains to database table file
       parseList "${adlistID}" "${saveLocation}" "${target}"
-      # Update date_updated field in gravity database table
+      # Compare lists, are they identical?
-      database_adlist_updated "${adlistID}"
+      compareLists "${adlistID}" "${saveLocation}"
+      # Update gravity database table (status and updated timestamp are set in
+      # compareLists)
+      database_adlist_number "${adlistID}"
+      done="true"
     else
       # Fall back to previously cached list if $patternBuffer is empty
-      echo -e "  ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
+      echo -e "  ${INFO} Received empty file"
     fi
-  else
+  fi
+
+  # Do we need to fall back to a cached list (if available)?
+  if [[ "${done}" != "true" ]]; then
     # Determine if cached list has read permission
     if [[ -r "${saveLocation}" ]]; then
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
       # Add domains to database table file
       parseList "${adlistID}" "${saveLocation}" "${target}"
+      database_adlist_number "${adlistID}"
+      database_adlist_status "${adlistID}" "3"
     else
       echo -e "  ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
+      # Manually reset these two numbers because we do not call parseList here
+      num_lines=0
+      num_invalid=0
+      database_adlist_number "${adlistID}"
+      database_adlist_status "${adlistID}" "4"
     fi
   fi
 }