Merge branch 'development' into tweak/reload_on_enable_disable

advanced/01-pihole.conf

@@ -24,6 +24,9 @@ addn-hosts=/etc/pihole/gravity.list
 addn-hosts=/etc/pihole/black.list
 addn-hosts=/etc/pihole/local.list
 
+user=pihole
+group=pihole
+
 domain-needed
 
 localise-queries
@@ -39,7 +42,7 @@ interface=@INT@
 
 cache-size=10000
 
-log-queries=extra
+log-queries
 log-facility=/var/log/pihole.log
 
 local-ttl=2

advanced/Scripts/piholeLogFlush.sh

@@ -58,6 +58,8 @@ else
     # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history)
     deleted=$(sqlite3 "${DBFILE}" "DELETE FROM queries WHERE timestamp >= strftime('%s','now')-86400; select changes() from queries limit 1")
 
+    # Restart pihole-FTL to force reloading history
+    sudo pihole restartdns
 fi
 
 if [[ "$@" != *"quiet"* ]]; then

advanced/Scripts/webpage.sh

@@ -541,8 +541,8 @@ clearAudit()
 }
 
 SetPrivacyLevel() {
-    # Set privacy level. Minimum is 0, maximum is 3
-    if [ "${args[2]}" -ge 0 ] && [ "${args[2]}" -le 3 ]; then
+    # Set privacy level. Minimum is 0, maximum is 4
+    if [ "${args[2]}" -ge 0 ] && [ "${args[2]}" -le 4 ]; then
         changeFTLsetting "PRIVACYLEVEL" "${args[2]}"
     fi
 }

advanced/Templates/pihole-FTL.service

@@ -26,17 +26,26 @@ start() {
   if is_running; then
     echo "pihole-FTL is already running"
   else
-    touch /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole.log
+    # Touch files to ensure they exist (create if non-existing, preserve if existing)
+    touch /var/log/pihole-FTL.log /var/log/pihole.log
+    touch /run/pihole-FTL.pid /run/pihole-FTL.port
+    touch /etc/pihole/dhcp.leases
     mkdir -p /var/run/pihole
     mkdir -p /var/log/pihole
     chown pihole:pihole /var/run/pihole /var/log/pihole
     rm /var/run/pihole/FTL.sock 2> /dev/null
-    chown pihole:pihole /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port
-    chown pihole:pihole /etc/pihole /etc/pihole/dhcp.leases /var/log/pihole.log
+    # Ensure that permissions are set so that pihole-FTL can edit all necessary files
+    chown pihole:pihole /run/pihole-FTL.pid /run/pihole-FTL.port
+    chown pihole:pihole /etc/pihole /etc/pihole/dhcp.leases
+    chown pihole:pihole /var/log/pihole-FTL.log /var/log/pihole.log
     chmod 0644 /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole.log
-    setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN+eip "$(which pihole-FTL)"
     echo "nameserver 127.0.0.1" | /sbin/resolvconf -a lo.piholeFTL
-    su -s /bin/sh -c "/usr/bin/pihole-FTL" "$FTLUSER"
+    if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN+eip "$(which pihole-FTL)"; then
+      su -s /bin/sh -c "/usr/bin/pihole-FTL" "$FTLUSER"
+    else
+      echo "Warning: Starting pihole-FTL as root because setting capabilities is not supported on this system"
+      pihole-FTL
+    fi
     echo
   fi
 }

automated install/basic-install.sh

@@ -1087,19 +1087,42 @@ chooseBlocklists() {
     # For each choice available,
     for choice in ${choices}
     do
-        # Set the values to true
-        case ${choice} in
-            StevenBlack  )  echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";;
-            MalwareDom   )  echo "https://mirror1.malwaredomains.com/files/justdomains" >> "${adlistFile}";;
-            Cameleon     )  echo "http://sysctl.org/cameleon/hosts" >> "${adlistFile}";;
-            ZeusTracker  )  echo "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist" >> "${adlistFile}";;
-            DisconTrack  )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" >> "${adlistFile}";;
-            DisconAd     )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt" >> "${adlistFile}";;
-            HostsFile    )  echo "https://hosts-file.net/ad_servers.txt" >> "${adlistFile}";;
-        esac
+        appendToListsFile choice
     done
 }
 
+# Accept a string parameter, it must be one of the default lists
+# This function allow to not duplicate code in chooseBlocklists and
+# in installDefaultBlocklists
+appendToListsFile() {
+    case $1 in
+        StevenBlack  )  echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";;
+        MalwareDom   )  echo "https://mirror1.malwaredomains.com/files/justdomains" >> "${adlistFile}";;
+        Cameleon     )  echo "http://sysctl.org/cameleon/hosts" >> "${adlistFile}";;
+        ZeusTracker  )  echo "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist" >> "${adlistFile}";;
+        DisconTrack  )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" >> "${adlistFile}";;
+        DisconAd     )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt" >> "${adlistFile}";;
+        HostsFile    )  echo "https://hosts-file.net/ad_servers.txt" >> "${adlistFile}";;
+    esac
+}
+
+# Used only in unattended setup
+# If there is already the adListFile, we keep it, else we create it using all default lists
+installDefaultBlocklists() {
+    # In unattended setup, could be useful to use userdefined blocklist.
+    # If this file exists, we avoid overriding it.
+    if [[ -f "${adlistFile}" ]]; then
+        return;
+    fi
+    appendToListsFile StevenBlack
+    appendToListsFile MalwareDom
+    appendToListsFile Cameleon
+    appendToListsFile ZeusTracker
+    appendToListsFile DisconTrack
+    appendToListsFile DisconAd
+    appendToListsFile HostsFile
+}
+
 # Check if /etc/dnsmasq.conf is from pi-hole.  If so replace with an original and install new in .d directory
 version_check_dnsmasq() {
     # Local, named variables
@@ -2086,12 +2109,15 @@ FTLinstall() {
                 fi
             fi
 
-            #ensure /etc/dnsmasq.conf contains `conf-dir=/etc/dnsmasq.d`
-            confdir="conf-dir=/etc/dnsmasq.d"
-            conffile="/etc/dnsmasq.conf"
-            if ! grep -q "$confdir" "$conffile"; then
-                echo "$confdir" >> "$conffile"
+            # Backup existing /etc/dnsmasq.conf if present and ensure that
+            # /etc/dnsmasq.conf contains only "conf-dir=/etc/dnsmasq.d"
+            local conffile="/etc/dnsmasq.conf"
+            if [[ -f "${conffile}" ]]; then
+                echo "  ${INFO} Backing up ${conffile} to ${conffile}.old"
+                mv "${conffile}" "${conffile}.old"
             fi
+            # Create /etc/dnsmasq.conf
+            echo "conf-dir=/etc/dnsmasq.d" > "${conffile}"
 
             return 0
         # Otherwise,
@@ -2099,7 +2125,7 @@ FTLinstall() {
             # the download failed, so just go back to the original directory
             popd > /dev/null || { echo "Unable to return to original directory after FTL binary download."; return 1; }
             echo -e "${OVER}  ${CROSS} ${str}"
-            echo -e "  ${COL_LIGHT_RED}Error: Download of binary from Github failed${COL_NC}"
+            echo -e "  ${COL_LIGHT_RED}Error: Download of ${url}/${binary} failed (checksum error)${COL_NC}"
             return 1
         fi
     # Otherwise,
@@ -2107,7 +2133,7 @@ FTLinstall() {
         popd > /dev/null || { echo "Unable to return to original directory after FTL binary download."; return 1; }
         echo -e "${OVER}  ${CROSS} ${str}"
         # The URL could not be found
-        echo -e "  ${COL_LIGHT_RED}Error: URL not found${COL_NC}"
+        echo -e "  ${COL_LIGHT_RED}Error: URL ${url}/${binary} not found${COL_NC}"
         return 1
     fi
 }
@@ -2391,6 +2417,8 @@ main() {
         # Let the user decide if they want query logging enabled...
         setLogging
     else
+        # Setup adlist file if not exists
+        installDefaultBlocklists
         # Source ${setupVars} to use predefined user variables in the functions
         source ${setupVars}
     fi
@@ -2459,8 +2487,11 @@ main() {
     # Start services
 
     # Enable FTL
-    start_service pihole-FTL
+    # Ensure the service is enabled before trying to start it
+    # Fixes a problem reported on Ubuntu 18.04 where trying to start
+    # the service before enabling causes installer to exit
     enable_service pihole-FTL
+    start_service pihole-FTL
 
     # Download and compile the aggregated block list
     runGravity

pihole

@@ -309,7 +309,7 @@ tailFunc() {
   # Colour everything else as gray
   tail -f /var/log/pihole.log | sed -E \
     -e "s,($(date +'%b %d ')| dnsmasq[.*[0-9]]),,g" \
-    -e "s,(.*(gravity.list|black.list| config ).* is (${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \
+    -e "s,(.*(gravity.list|black.list|regex.list| config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \
     -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \
     -e "s,.*,${COL_GRAY}&${COL_NC},"
   exit 0

test/test_automated_install.py

@@ -501,8 +501,10 @@ def test_FTL_download_unknown_fails_no_errors(Pihole):
     ''')
     expected_stdout = cross_box + ' Downloading and Installing FTL'
     assert expected_stdout in download_binary.stdout
-    error = 'Error: URL not found'
-    assert error in download_binary.stdout
+    error1 = 'Error: URL https://github.com/pi-hole/FTL/releases/download/'
+    assert error1 in download_binary.stdout
+    error2 = 'not found'
+    assert error2 in download_binary.stdout
 
 
 def test_FTL_binary_installed_and_responsive_no_errors(Pihole):