Merge remote-tracking branch 'origin/development' into ipv6_brackets

Signed-off-by: Patrik Cyvoct <patrik@ptrk.io>

advanced/01-pihole.conf

@@ -39,7 +39,7 @@ interface=@INT@
 
 cache-size=10000
 
-log-queries=extra
+log-queries
 log-facility=/var/log/pihole.log
 
 local-ttl=2

advanced/Scripts/chronometer.sh

@@ -478,7 +478,7 @@ chronoFunc() {
             ${COL_LIGHT_RED}Press Ctrl-C to exit${COL_NC}
             ${COL_DARK_GRAY}$scr_line_str${COL_NC}"
         else
-        echo -e "|¯¯¯(¯)_|¯|_  ___|¯|___$phc_ver_str| ¯_/¯|_| ' \\/ _ \\ / -_)$lte_ver_str|_| |_| |_||_\\___/_\\___|$ftl_ver_str ${COL_DARK_GRAY}$scr_line_str${COL_NC}"
+        echo -e "|¯¯¯(¯)_|¯|_  ___|¯|___$phc_ver_str\\n| ¯_/¯|_| ' \\/ _ \\ / -_)$lte_ver_str\\n|_| |_| |_||_\\___/_\\___|$ftl_ver_str\\n ${COL_DARK_GRAY}$scr_line_str${COL_NC}"
         fi
 
         printFunc "  Hostname: " "$sys_name" "$host_info"

advanced/Scripts/piholeLogFlush.sh

@@ -58,6 +58,8 @@ else
     # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history)
     deleted=$(sqlite3 "${DBFILE}" "DELETE FROM queries WHERE timestamp >= strftime('%s','now')-86400; select changes() from queries limit 1")
 
+    # Restart pihole-FTL to force reloading history
+    sudo pihole restartdns
 fi
 
 if [[ "$@" != *"quiet"* ]]; then

advanced/Scripts/webpage.sh

@@ -525,14 +525,24 @@ Teleporter() {
     php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-teleporter_${datetimestamp}.zip"
 }
 
-audit()
+addAudit()
 {
-    echo "${args[2]}" >> /etc/pihole/auditlog.list
+    shift # skip "-a"
+    shift # skip "audit"
+    for var in "$@"
+    do
+        echo "${var}" >> /etc/pihole/auditlog.list
+    done
+}
+
+clearAudit()
+{
+    echo -n "" > /etc/pihole/auditlog.list
 }
 
 SetPrivacyLevel() {
-    # Set privacy level. Minimum is 0, maximum is 3
-    if [ "${args[2]}" -ge 0 ] && [ "${args[2]}" -le 3 ]; then
+    # Set privacy level. Minimum is 0, maximum is 4
+    if [ "${args[2]}" -ge 0 ] && [ "${args[2]}" -le 4 ]; then
         changeFTLsetting "PRIVACYLEVEL" "${args[2]}"
     fi
 }
@@ -565,7 +575,8 @@ main() {
         "-i" | "interface"    ) SetListeningMode "$@";;
         "-t" | "teleporter"   ) Teleporter;;
         "adlist"              ) CustomizeAdLists;;
-        "audit"               ) audit;;
+        "audit"               ) addAudit "$@";;
+        "clearaudit"          ) clearAudit;;
         "-l" | "privacylevel" ) SetPrivacyLevel;;
         *                     ) helpFunc;;
     esac

advanced/Templates/pihole-FTL.service

@@ -26,17 +26,26 @@ start() {
   if is_running; then
     echo "pihole-FTL is already running"
   else
-    touch /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole.log
+    # Touch files to ensure they exist (create if non-existing, preserve if existing)
+    touch /var/log/pihole-FTL.log /var/log/pihole.log
+    touch /run/pihole-FTL.pid /run/pihole-FTL.port
+    touch /etc/pihole/dhcp.leases
     mkdir -p /var/run/pihole
     mkdir -p /var/log/pihole
     chown pihole:pihole /var/run/pihole /var/log/pihole
     rm /var/run/pihole/FTL.sock 2> /dev/null
-    chown pihole:pihole /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port
-    chown pihole:pihole /etc/pihole /etc/pihole/dhcp.leases /var/log/pihole.log
+    # Ensure that permissions are set so that pihole-FTL can edit all necessary files
+    chown pihole:pihole /run/pihole-FTL.pid /run/pihole-FTL.port
+    chown pihole:pihole /etc/pihole /etc/pihole/dhcp.leases
+    chown pihole:pihole /var/log/pihole-FTL.log /var/log/pihole.log
     chmod 0644 /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole.log
-    setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN+eip "$(which pihole-FTL)"
     echo "nameserver 127.0.0.1" | /sbin/resolvconf -a lo.piholeFTL
-    su -s /bin/sh -c "/usr/bin/pihole-FTL" "$FTLUSER"
+    if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN+eip "$(which pihole-FTL)"; then
+      su -s /bin/sh -c "/usr/bin/pihole-FTL" "$FTLUSER"
+    else
+      echo "Warning: Starting pihole-FTL as root because setting capabilities is not supported on this system"
+      pihole-FTL
+    fi
     echo
   fi
 }
@@ -78,7 +87,7 @@ status() {
     echo "[    ] pihole-FTL is not running"
     exit 1
   fi
-}  
+}
 
 
 ### main logic ###

advanced/index.php

@@ -8,7 +8,7 @@
 
 // Sanitise HTTP_HOST output
 $serverName = htmlspecialchars($_SERVER["HTTP_HOST"]);
-// Remove external ipv6 brackets if any 
+// Remove external ipv6 brackets if any
 $serverName = preg_replace('/^\[(.*)\]$/', '${1}', $serverName);
 
 if (!is_file("/etc/pihole/setupVars.conf"))

automated install/basic-install.sh

@@ -38,14 +38,14 @@ lighttpdConfig=/etc/lighttpd/lighttpd.conf
 # This is a file used for the colorized output
 coltable=/opt/pihole/COL_TABLE
 
-# We store several other folders and
+# We store several other directories and
 webInterfaceGitUrl="https://github.com/pi-hole/AdminLTE.git"
 webInterfaceDir="/var/www/html/admin"
 piholeGitUrl="https://github.com/pi-hole/pi-hole.git"
 PI_HOLE_LOCAL_REPO="/etc/.pihole"
 # These are the names of pi-holes files, stored in an array
 PI_HOLE_FILES=(chronometer list piholeDebug piholeLogFlush setupLCD update version gravity uninstall webpage)
-# This folder is where the Pi-hole scripts will be installed
+# This directory is where the Pi-hole scripts will be installed
 PI_HOLE_INSTALL_DIR="/opt/pihole"
 PI_HOLE_CONFIG_DIR="/etc/pihole"
 useUpdateVars=false
@@ -320,7 +320,7 @@ else
 fi
 }
 
-# A function for checking if a folder is a git repository
+# A function for checking if a directory is a git repository
 is_repo() {
     # Use a named, local variable instead of the vague $1, which is the first argument passed to this function
     # These local variables should always be lowercase
@@ -335,7 +335,7 @@ is_repo() {
     if [[ -d "${directory}" ]]; then
         # move into the directory
         cd "${directory}"
-        # Use git to check if the folder is a repo
+        # Use git to check if the directory is a repo
         # git -C is not used here to support git versions older than 1.8.4
         git status --short &> /dev/null || rc=$?
     # If the command was not successful,
@@ -1087,19 +1087,42 @@ chooseBlocklists() {
     # For each choice available,
     for choice in ${choices}
     do
-        # Set the values to true
-        case ${choice} in
-            StevenBlack  )  echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";;
-            MalwareDom   )  echo "https://mirror1.malwaredomains.com/files/justdomains" >> "${adlistFile}";;
-            Cameleon     )  echo "http://sysctl.org/cameleon/hosts" >> "${adlistFile}";;
-            ZeusTracker  )  echo "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist" >> "${adlistFile}";;
-            DisconTrack  )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" >> "${adlistFile}";;
-            DisconAd     )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt" >> "${adlistFile}";;
-            HostsFile    )  echo "https://hosts-file.net/ad_servers.txt" >> "${adlistFile}";;
-        esac
+        appendToListsFile choice
     done
 }
 
+# Accept a string parameter, it must be one of the default lists
+# This function allow to not duplicate code in chooseBlocklists and 
+# in installDefaultBlocklists
+appendToListsFile() {
+    case $1 in
+        StevenBlack  )  echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";;
+        MalwareDom   )  echo "https://mirror1.malwaredomains.com/files/justdomains" >> "${adlistFile}";;
+        Cameleon     )  echo "http://sysctl.org/cameleon/hosts" >> "${adlistFile}";;
+        ZeusTracker  )  echo "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist" >> "${adlistFile}";;
+        DisconTrack  )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" >> "${adlistFile}";;
+        DisconAd     )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt" >> "${adlistFile}";;
+        HostsFile    )  echo "https://hosts-file.net/ad_servers.txt" >> "${adlistFile}";;
+    esac
+}
+
+# Used only in unattended setup
+# If there is already the adListFile, we keep it, else we create it using all default lists
+installDefaultBlocklists() {
+    # In unattended setup, could be useful to use userdefined blocklist.
+    # If this file exists, we avoid overriding it.
+    if [[ -f "${adlistFile}" ]]; then
+        return;
+    fi  
+    appendToListsFile StevenBlack
+    appendToListsFile MalwareDom
+    appendToListsFile Cameleon
+    appendToListsFile ZeusTracker
+    appendToListsFile DisconTrack
+    appendToListsFile DisconAd
+    appendToListsFile HostsFile
+}
+
 # Check if /etc/dnsmasq.conf is from pi-hole.  If so replace with an original and install new in .d directory
 version_check_dnsmasq() {
     # Local, named variables
@@ -2099,7 +2122,7 @@ FTLinstall() {
             # the download failed, so just go back to the original directory
             popd > /dev/null || { echo "Unable to return to original directory after FTL binary download."; return 1; }
             echo -e "${OVER}  ${CROSS} ${str}"
-            echo -e "  ${COL_LIGHT_RED}Error: Download of binary from Github failed${COL_NC}"
+            echo -e "  ${COL_LIGHT_RED}Error: Download of ${url}/${binary} failed (checksum error)${COL_NC}"
             return 1
         fi
     # Otherwise,
@@ -2107,7 +2130,7 @@ FTLinstall() {
         popd > /dev/null || { echo "Unable to return to original directory after FTL binary download."; return 1; }
         echo -e "${OVER}  ${CROSS} ${str}"
         # The URL could not be found
-        echo -e "  ${COL_LIGHT_RED}Error: URL not found${COL_NC}"
+        echo -e "  ${COL_LIGHT_RED}Error: URL ${url}/${binary} not found${COL_NC}"
         return 1
     fi
 }
@@ -2151,11 +2174,6 @@ get_binary_name() {
             # set the binary to be used
             binary="pihole-FTL-arm-linux-gnueabi"
         fi
-    elif [[ "${machine}" == "ppc" ]]; then
-        # PowerPC
-        echo -e "${OVER}  ${TICK} Detected PowerPC architecture"
-        # set the binary to be used
-        binary="pihole-FTL-powerpc-linux-gnu"
     elif [[ "${machine}" == "x86_64" ]]; then
         # This gives the architecture of packages dpkg installs (for example, "i386")
         local dpkgarch
@@ -2347,6 +2365,8 @@ main() {
             echo -e "  ${INFO} Performing unattended setup, no whiptail dialogs will be displayed"
             # Use the setup variables
             useUpdateVars=true
+            # also disable debconf-apt-progress dialogs
+            export DEBIAN_FRONTEND="noninteractive"
         # Otherwise,
         else
             # show the available options (repair/reconfigure)
@@ -2394,6 +2414,8 @@ main() {
         # Let the user decide if they want query logging enabled...
         setLogging
     else
+        # Setup adlist file if not exists
+        installDefaultBlocklists
         # Source ${setupVars} to use predefined user variables in the functions
         source ${setupVars}
     fi
@@ -2461,9 +2483,12 @@ main() {
     echo -e "  ${INFO} Restarting services..."
     # Start services
 
-    # Enable FTL
-    start_service pihole-FTL
+    # Enable FTL 
+    # Ensure the service is enabled before trying to start it
+    # Fixes a problem reported on Ubuntu 18.04 where trying to start
+    # the service before enabling causes installer to exit
     enable_service pihole-FTL
+    start_service pihole-FTL
 
     # Download and compile the aggregated block list
     runGravity

automated install/uninstall.sh

@@ -106,7 +106,7 @@ removeNoPurge() {
     ${SUDO} rm -rf /var/www/html/pihole &> /dev/null
     ${SUDO} rm -f /var/www/html/index.lighttpd.orig &> /dev/null
 
-    # If the web directory is empty after removing these files, then the parent html folder can be removed.
+    # If the web directory is empty after removing these files, then the parent html directory can be removed.
     if [ -d "/var/www/html" ]; then
         if [[ ! "$(ls -A /var/www/html)" ]]; then
             ${SUDO} rm -rf /var/www/html &> /dev/null

gravity.sh

@@ -464,7 +464,7 @@ gravity_ShowBlockCount() {
   fi
 
   if [[ -f "${regexFile}" ]]; then
-    num=$(grep -c "^(?!#)" "${regexFile}")
+    num=$(grep -cv "^#" "${regexFile}")
     echo -e "  ${INFO} Number of regex filters: ${num}"
   fi
 }

pihole

@@ -303,7 +303,7 @@ tailFunc() {
   # Colour everything else as gray
   tail -f /var/log/pihole.log | sed -E \
     -e "s,($(date +'%b %d ')| dnsmasq[.*[0-9]]),,g" \
-    -e "s,(.*(gravity.list|black.list| config ).* is (${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \
+    -e "s,(.*(gravity.list|black.list|regex.list| config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \
     -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \
     -e "s,.*,${COL_GRAY}&${COL_NC},"
   exit 0

test/test_automated_install.py

@@ -501,8 +501,10 @@ def test_FTL_download_unknown_fails_no_errors(Pihole):
     ''')
     expected_stdout = cross_box + ' Downloading and Installing FTL'
     assert expected_stdout in download_binary.stdout
-    error = 'Error: URL not found'
-    assert error in download_binary.stdout
+    error1 = 'Error: URL https://github.com/pi-hole/FTL/releases/download/'
+    assert error1 in download_binary.stdout
+    error2 = 'not found'
+    assert error2 in download_binary.stdout
 
 
 def test_FTL_binary_installed_and_responsive_no_errors(Pihole):