Merge branch 'development' into packages

.github/workflows/test.yml

@@ -65,7 +65,6 @@ jobs:
             debian_12,
             ubuntu_20,
             ubuntu_22,
-            ubuntu_23,
             ubuntu_24,
             centos_9,
             fedora_40,

advanced/Scripts/database_migration/gravity-db.sh

@@ -13,10 +13,9 @@
 readonly scriptPath="/etc/.pihole/advanced/Scripts/database_migration/gravity"
 
 upgrade_gravityDB(){
-    local database piholeDir auditFile version
+    local database piholeDir version
     database="${1}"
     piholeDir="${2}"
-    auditFile="${piholeDir}/auditlog.list"
 
     # Exit early if the database does not exist (e.g. in CI tests)
     if [[ ! -f "${database}" ]]; then
@@ -27,18 +26,11 @@ upgrade_gravityDB(){
     version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
 
     if [[ "$version" == "1" ]]; then
-        # This migration script upgrades the gravity.db file by
-        # adding the domain_audit table
+        # This migration script upgraded the gravity.db file by
+        # adding the domain_audit table. It is now a no-op
         echo -e "  ${INFO} Upgrading gravity database from version 1 to 2"
         pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql"
         version=2
-
-        # Store audit domains in database table
-        if [ -e "${auditFile}" ]; then
-            echo -e "  ${INFO} Migrating content of ${auditFile} into new database"
-            # database_table_from_file is defined in gravity.sh
-            database_table_from_file "domain_audit" "${auditFile}"
-        fi
     fi
     if [[ "$version" == "2" ]]; then
         # This migration script upgrades the gravity.db file by

advanced/Scripts/database_migration/gravity/1_to_2.sql

@@ -2,13 +2,6 @@
 
 BEGIN TRANSACTION;
 
-CREATE TABLE domain_audit
-(
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    domain TEXT UNIQUE NOT NULL,
-    date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
-);
-
 UPDATE info SET value = 2 WHERE property = 'version';
 
 COMMIT;

advanced/Scripts/piholeARPTable.sh

@@ -32,7 +32,7 @@ flushARP(){
     fi
 
     # Stop FTL to prevent database access
-    if ! output=$(pihole-FTL service stop 2>&1); then
+    if ! output=$(service pihole-FTL stop 2>&1); then
         echo -e "${OVER}  ${CROSS} Failed to stop FTL"
         echo "  Output: ${output}"
         return 1
@@ -64,7 +64,7 @@ flushARP(){
     fi
 
     # Start FTL again
-    if ! output=$(pihole-FTL service restart 2>&1); then
+    if ! output=$(service pihole-FTL restart 2>&1); then
         echo -e "${OVER}  ${CROSS} Failed to restart FTL"
         echo "  Output: ${output}"
         return 1

advanced/Templates/gravity.db.sql

@@ -68,13 +68,6 @@ CREATE TABLE info
 
 INSERT INTO "info" VALUES('version','19');
 
-CREATE TABLE domain_audit
-(
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    domain TEXT UNIQUE NOT NULL,
-    date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
-);
-
 CREATE TABLE domainlist_by_group
 (
     domainlist_id INTEGER NOT NULL REFERENCES domainlist (id),

advanced/Templates/gravity_copy.sql

@@ -9,7 +9,6 @@ DROP TRIGGER tr_client_add;
 DROP TRIGGER tr_adlist_add;
 
 INSERT OR REPLACE INTO "group" SELECT * FROM OLD."group";
-INSERT OR REPLACE INTO domain_audit SELECT * FROM OLD.domain_audit;
 
 INSERT OR REPLACE INTO domainlist SELECT * FROM OLD.domainlist;
 DELETE FROM OLD.domainlist_by_group WHERE domainlist_id NOT IN (SELECT id FROM OLD.domainlist);

gravity.sh

@@ -59,14 +59,24 @@ gravityTEMPfile="${GRAVITYDB}_temp"
 gravityDIR="$(dirname -- "${gravityDBfile}")"
 gravityOLDfile="${gravityDIR}/gravity_old.db"
 
+fix_owner_permissions() {
+  # Fix ownership and permissions for the specified file
+  # User and group are set to pihole:pihole
+  # Permissions are set to 664 (rw-rw-r--)
+  chown pihole:pihole "${1}"
+  chmod 664 "${1}"
+
+  # Ensure the containing directory is group writable
+  chmod g+w "$(dirname -- "${1}")"
+}
+
 # Generate new SQLite3 file from schema template
 generate_gravity_database() {
   if ! pihole-FTL sqlite3 -ni "${gravityDBfile}" <"${gravityDBschema}"; then
     echo -e "   ${CROSS} Unable to create ${gravityDBfile}"
     return 1
   fi
-  chown pihole:pihole "${gravityDBfile}"
-  chmod g+w "${piholeDir}" "${gravityDBfile}"
+  fix_owner_permissions "${gravityDBfile}"
 }
 
 # Build gravity tree
@@ -172,10 +182,7 @@ database_table_from_file() {
   grep -v '^ *#' <"${src}" | while IFS= read -r domain; do
     # Only add non-empty lines
     if [[ -n "${domain}" ]]; then
-      if [[ "${table}" == "domain_audit" ]]; then
-        # domain_audit table format (no enable or modified fields)
-        echo "${rowid},\"${domain}\",${timestamp}" >>"${tmpFile}"
-      elif [[ "${table}" == "adlist" ]]; then
+      if [[ "${table}" == "adlist" ]]; then
         # Adlist table format
         echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0,0" >>"${tmpFile}"
       else
@@ -351,7 +358,7 @@ gravity_DownloadBlocklists() {
     unset sources
   fi
 
-  local url domain str target compression adlist_type
+  local url domain str target compression adlist_type directory
   echo ""
 
   # Prepare new gravity database
@@ -416,6 +423,24 @@ gravity_DownloadBlocklists() {
     saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}"
     activeDomains[$i]="${saveLocation}"
 
+    # Check if we can write to the save location file without actually creating
+    # it (in case it doesn't exist)
+    # First, check if the directory is writable
+    directory="$(dirname -- "${saveLocation}")"
+    if [ ! -w "${directory}" ]; then
+      echo -e "  ${CROSS} Unable to write to ${directory}"
+      echo "      Please run pihole -g as root"
+      echo ""
+      continue
+    fi
+    # Then, check if the file is writable (if it exists)
+    if [ -e "${saveLocation}" ] && [ ! -w "${saveLocation}" ]; then
+      echo -e "  ${CROSS} Unable to write to ${saveLocation}"
+      echo "      Please run pihole -g as root"
+      echo ""
+      continue
+    fi
+
     echo -e "  ${INFO} Target: ${url}"
     local regex check_url
     # Check for characters NOT allowed in URLs
@@ -444,6 +469,7 @@ compareLists() {
     if ! sha1sum --check --status --strict "${target}.sha1"; then
       # The list changed upstream, we need to update the checksum
       sha1sum "${target}" >"${target}.sha1"
+      fix_owner_permissions "${target}.sha1"
       echo "  ${INFO} List has been updated"
       database_adlist_status "${adlistID}" "1"
     else
@@ -453,6 +479,7 @@ compareLists() {
   else
     # No checksum available, create one for comparing on the next run
     sha1sum "${target}" >"${target}.sha1"
+    fix_owner_permissions "${target}.sha1"
     # We assume here it was changed upstream
     database_adlist_status "${adlistID}" "1"
   fi
@@ -714,7 +741,7 @@ gravity_ParseFileIntoDomains() {
     -e 's/^.*\s+//g' \
     -e '/^$/d' "${destination}"
 
-  chmod 644 "${destination}"
+  fix_owner_permissions "${destination}"
 }
 
 # Report number of entries in a table
@@ -999,8 +1026,7 @@ fi
 update_gravity_timestamp
 
 # Ensure proper permissions are set for the database
-chown pihole:pihole "${gravityTEMPfile}"
-chmod g+w "${piholeDir}" "${gravityTEMPfile}"
+fix_owner_permissions "${gravityTEMPfile}"
 
 # Build the tree
 timeit gravity_build_tree

test/requirements.txt

@@ -1,5 +1,5 @@
 pyyaml == 6.0.2
-pytest == 8.3.3
+pytest == 8.3.4
 pytest-xdist == 3.6.1
 pytest-testinfra == 10.1.1
 tox == 4.23.2

test/tox.ubuntu_23.ini

@@ -1,10 +0,0 @@
-[tox]
-envlist = py3
-
-[testenv:py3]
-allowlist_externals = docker
-deps = -rrequirements.txt
-setenv =
-    COLUMNS=120
-commands = docker buildx build --load --progress plain -f _ubuntu_23.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py