Merge pull request #2056 from pi-hole/release/v3.3.1

Pi-hole core v3.3.1

.pullapprove.yml

@@ -1,38 +0,0 @@
-version: 2
-
-always_pending:
-  title_regex: '(WIP|wip)'
-  labels:
-    - wip
-  explanation: 'This PR is a work in progress...'
-
-group_defaults:
-  reset_on_push:
-    enabled: true
-  reject_value: -2
-  approve_regex: '^(Approved|:shipit:|:\+1:|Engage|:taco:)'
-  reject_regex: '^(Rejected|:-1:|Borg)'
-  author_approval:
-    auto: true
-
-
-groups:
-  development:
-    approve_by_comment:
-      enabled: true
-    conditions:
-      branches:
-        - development
-    required: 2
-    teams:
-      - approvers
-
-  master:
-    approve_by_comment:
-      enabled: true
-    conditions:
-      branches:
-        - master
-    required: 4
-    teams:
-      - approvers

advanced/Scripts/list.sh

@@ -10,9 +10,9 @@
 
 # Globals
 basename=pihole
-piholeDir=/etc/${basename}
+piholeDir=/etc/"${basename}"
-whitelist=${piholeDir}/whitelist.txt
+whitelist="${piholeDir}"/whitelist.txt
-blacklist=${piholeDir}/blacklist.txt
+blacklist="${piholeDir}"/blacklist.txt
 readonly wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf"
 reload=false
 addmode=true
@@ -80,8 +80,13 @@ HandleOther() {
 
 PoplistFile() {
   # Check whitelist file exists, and if not, create it
-  if [[ ! -f ${whitelist} ]]; then
+  if [[ ! -f "${whitelist}" ]]; then
-    touch ${whitelist}
+    touch "${whitelist}"
+  fi
+
+  # Check blacklist file exists, and if not, create it
+  if [[ ! -f "${blacklist}" ]]; then
+    touch "${blacklist}"
   fi
 
   for dom in "${domList[@]}"; do

advanced/Scripts/piholeCheckout.sh

@@ -256,6 +256,7 @@ checkout() {
     get_binary_name
     local path
     path="development/${binary}"
+    echo "development" > /etc/pihole/ftlbranch
     FTLinstall "${binary}" "${path}"
   elif [[ "${1}" == "master" ]] ; then
     # Shortcut to check out master branches
@@ -270,6 +271,7 @@ checkout() {
     get_binary_name
     local path
     path="master/${binary}"
+    echo "master" > /etc/pihole/ftlbranch
     FTLinstall "${binary}" "${path}"
   elif [[ "${1}" == "core" ]] ; then
     str="Fetching branches from ${piholeGitUrl}"
@@ -332,6 +334,7 @@ checkout() {
 
     if check_download_exists "$path"; then
         echo "  ${TICK} Branch ${2} exists"
+        echo "${2}" > /etc/pihole/ftlbranch
         FTLinstall "${binary}" "${path}"
     else
         echo "  ${CROSS} Requested branch \"${2}\" is not available"

advanced/Scripts/piholeLogFlush.sh

@@ -11,6 +11,17 @@
 colfile="/opt/pihole/COL_TABLE"
 source ${colfile}
 
+# Determine database location
+# Obtain DBFILE=... setting from pihole-FTL.db
+# Constructed to return nothing when
+# a) the setting is not present in the config file, or
+# b) the setting is commented out (e.g. "#DBFILE=...")
+DBFILE="$(sed -n -e 's/^\s^.DBFILE\s*=\s*//p' /etc/pihole/pihole-FTL.conf)"
+# Test for empty string. Use standard path in this case.
+if [ -z "$DBFILE" ]; then
+  DBFILE="/etc/pihole/pihole-FTL.db"
+fi
+
 if [[ "$@" != *"quiet"* ]]; then
   echo -ne "  ${INFO} Flushing /var/log/pihole.log ..."
 fi
@@ -41,8 +52,12 @@ else
       echo " " > /var/log/pihole.log.1
     fi
   fi
+  # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history)
+  deleted=$(sqlite3 "${DBFILE}" "DELETE FROM queries WHERE timestamp >= strftime('%s','now')-86400; select changes() from queries limit 1")
+
 fi
 
 if [[ "$@" != *"quiet"* ]]; then
   echo -e "${OVER}  ${TICK} Flushed /var/log/pihole.log"
+  echo -e "  ${TICK} Deleted ${deleted} queries from database"
 fi

advanced/Scripts/webpage.sh

@@ -182,6 +182,10 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423
 
 		add_dnsmasq_setting "interface" "${PIHOLE_INTERFACE}"
 	fi
+	if [[ "${CONDITIONAL_FORWARDING}" == true ]]; then
+		add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_DOMAIN}/${CONDITIONAL_FORWARDING_IP}"
+		add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_REVERSE}/${CONDITIONAL_FORWARDING_IP}"
+	fi
 
 }
 
@@ -211,6 +215,17 @@ SetDNSServers() {
 	else
 		change_setting "DNSSEC" "false"
 	fi
+	if [[ "${args[6]}" == "conditional_forwarding" ]]; then
+		change_setting "CONDITIONAL_FORWARDING" "true"
+		change_setting "CONDITIONAL_FORWARDING_IP" "${args[7]}"
+		change_setting "CONDITIONAL_FORWARDING_DOMAIN" "${args[8]}"
+		change_setting "CONDITIONAL_FORWARDING_REVERSE" "${args[9]}"
+	else
+		change_setting "CONDITIONAL_FORWARDING" "false"
+		delete_setting "CONDITIONAL_FORWARDING_IP"
+		delete_setting "CONDITIONAL_FORWARDING_DOMAIN"
+		delete_setting "CONDITIONAL_FORWARDING_REVERSE"
+	fi
 
 	ProcessDNSSettings
 

advanced/index.php

@@ -98,9 +98,6 @@ if ($serverName === "pi.hole") {
 
 /* Start processing Block Page from here */
 
-// Determine placeholder text based off $svPasswd presence
-$wlPlaceHolder = empty($svPasswd) ? "No admin password set" : "Javascript disabled";
-
 // Define admin email address text based off $svEmail presence
 $bpAskAdmin = !empty($svEmail) ? '<a href="mailto:'.$svEmail.'?subject=Site Blocked: '.$serverName.'"></a>' : "<span/>";
 
@@ -236,11 +233,21 @@ setHeader();
     window.onload = function () {
       <?php
       // Remove href fallback from "Back to safety" button
-      if ($featuredTotal > 0) echo '$("#bpBack").removeAttr("href");';
+      if ($featuredTotal > 0) {
-      // Enable whitelisting if $svPasswd is present & JS is available
+        echo '$("#bpBack").removeAttr("href");';
-      if (!empty($svPasswd) && $featuredTotal > 0) {
+
-          echo '$("#bpWLPassword, #bpWhitelist").prop("disabled", false);';
+        // Enable whitelisting if JS is available
+        echo '$("#bpWhitelist").prop("disabled", false);';
+
+        // Enable password input if necessary
+        if (!empty($svPasswd)) {
           echo '$("#bpWLPassword").attr("placeholder", "Password");';
+          echo '$("#bpWLPassword").prop("disabled", false);';
+        }
+        // Otherwise hide the input
+        else {
+          echo '$("#bpWLPassword").hide();';
+        }
       }
       ?>
     }
@@ -294,7 +301,7 @@ setHeader();
 
     <form id="bpWLButtons" class="buttons">
       <input id="bpWLDomain" type="text" value="<?=$serverName ?>" disabled/>
-      <input id="bpWLPassword" type="password" placeholder="<?=$wlPlaceHolder ?>" disabled/><button id="bpWhitelist" type="button" disabled></button>
+      <input id="bpWLPassword" type="password" placeholder="Javascript disabled" disabled/><button id="bpWhitelist" type="button" disabled></button>
     </form>
   </div>
 </main>

automated install/basic-install.sh

@@ -23,7 +23,7 @@ set -e
 
 ######## VARIABLES #########
 # For better maintainability, we store as much information that can change in variables
-# This allows us to make a change in one place that can propogate to all instances of the variable
+# This allows us to make a change in one place that can propagate to all instances of the variable
 # These variables should all be GLOBAL variables, written in CAPS
 # Local variables will be in lowercase and will exist only within functions
 # It's still a work in progress, so you may see some variance in this guideline until it is complete
@@ -43,7 +43,7 @@ webInterfaceGitUrl="https://github.com/pi-hole/AdminLTE.git"
 webInterfaceDir="/var/www/html/admin"
 piholeGitUrl="https://github.com/pi-hole/pi-hole.git"
 PI_HOLE_LOCAL_REPO="/etc/.pihole"
-# These are the names of piholes files, stored in an array
+# These are the names of pi-holes files, stored in an array
 PI_HOLE_FILES=(chronometer list piholeDebug piholeLogFlush setupLCD update version gravity uninstall webpage)
 # This folder is where the Pi-hole scripts will be installed
 PI_HOLE_INSTALL_DIR="/opt/pihole"
@@ -81,7 +81,7 @@ runUnattended=false
 if [[ -f "${coltable}" ]]; then
   # source it
   source ${coltable}
-# Othwerise,
+# Otherwise,
 else
   # Set these values so the installer can still run in color
   COL_NC='\e[0m' # No Color
@@ -163,7 +163,7 @@ if command -v apt-get &> /dev/null; then
   # These programs are stored in an array so they can be looped through later
   INSTALLER_DEPS=(apt-utils dialog debconf dhcpcd5 git ${iproute_pkg} whiptail)
   # Pi-hole itself has several dependencies that also need to be installed
-  PIHOLE_DEPS=(bc cron curl dnsmasq dnsutils iputils-ping lsof netcat sudo unzip wget idn2)
+  PIHOLE_DEPS=(bc cron curl dnsmasq dnsutils iputils-ping lsof netcat sudo unzip wget idn2 sqlite3)
   # The Web dashboard has some that also need to be installed
   # It's useful to separate the two since our repos are also setup as "Core" code and "Web" code
   PIHOLE_WEB_DEPS=(lighttpd ${phpVer}-common ${phpVer}-cgi ${phpVer}-${phpSqlite})
@@ -771,6 +771,7 @@ setDNS() {
       Comodo ""
       DNSWatch ""
       Quad9 ""
+      FamilyShield ""
       Custom "")
   # In a whiptail dialog, show the options
   DNSchoices=$(whiptail --separate-output --menu "Select Upstream DNS Provider. To use your own, select Custom." ${r} ${c} 7 \
@@ -817,6 +818,11 @@ setDNS() {
       PIHOLE_DNS_1="9.9.9.9"
       PIHOLE_DNS_2="149.112.112.112"
       ;;
+    FamilyShield)
+      echo "FamilyShield servers"
+      PIHOLE_DNS_1="208.67.222.123"
+      PIHOLE_DNS_2="208.67.220.123"
+      ;;
     Custom)
       # Until the DNS settings are selected,
       until [[ "${DNSSettingsCorrect}" = True ]]; do
@@ -918,7 +924,7 @@ setLogging() {
     esac
 }
 
-# Funtion to ask the user if they want to install the dashboard
+# Function to ask the user if they want to install the dashboard
 setAdminFlag() {
   # Local, named variables
   local WebToggleCommand
@@ -946,7 +952,7 @@ setAdminFlag() {
     esac
 }
 
-# Check if /etc/dnsmasq.conf is from pihole.  If so replace with an original and install new in .d directory
+# Check if /etc/dnsmasq.conf is from pi-hole.  If so replace with an original and install new in .d directory
 version_check_dnsmasq() {
   # Local, named variables
   local dnsmasq_conf="/etc/dnsmasq.conf"
@@ -1734,17 +1740,14 @@ clone_or_update_repos() {
   fi
 }
 
-# Download and install FTL binary
+# Download FTL binary to random temp directory and install FTL binary
 FTLinstall() {
   # Local, named variables
   local binary="${1}"
   local latesttag
-  local orig_dir
   local str="Downloading and Installing FTL"
   echo -ne "  ${INFO} ${str}..."
 
-  # Get the current working directory
-  orig_dir="${PWD}"
   # Find the latest version tag for FTL
   latesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep "Location" | awk -F '/' '{print $NF}')
   # Tags should always start with v, check for that.
@@ -1754,42 +1757,44 @@ FTLinstall() {
     return 1
   fi
 
-  # If the download worked,
+  # Move into the temp ftl directory
-  if curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${latesttag%$'\r'}/${binary}" -o "/tmp/${binary}"; then
+  pushd "$(mktemp -d)" || { echo "Unable to make temporary directory for FTL binary download"; return 1; }
-    # get sha1 of the binary we just downloaded for verification.
+
-    curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${latesttag%$'\r'}/${binary}.sha1" -o "/tmp/${binary}.sha1"
+  # Always replace pihole-FTL.service
+  install -T -m 0755 "${PI_HOLE_LOCAL_REPO}/advanced/pihole-FTL.service" "/etc/init.d/pihole-FTL"
+
+  # If the download worked,
+  if curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${latesttag%$'\r'}/${binary}" -o "${binary}"; then
+    # get sha1 of the binary we just downloaded for verification.
+    curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${latesttag%$'\r'}/${binary}.sha1" -o "${binary}.sha1"
 
-    # Move into the temp directory
-    cd /tmp
     # If we downloaded binary file (as opposed to text),
     if sha1sum --status --quiet -c "${binary}".sha1; then
       echo -n "transferred... "
       # Stop FTL
       stop_service pihole-FTL &> /dev/null
       # Install the new version with the correct permissions
-      install -T -m 0755 /tmp/${binary} /usr/bin/pihole-FTL
+      install -T -m 0755 "${binary}" /usr/bin/pihole-FTL
-      # Remove the tempoary file
-      rm /tmp/${binary} /tmp/${binary}.sha1
       # Move back into the original directory the user was in
-      cd "${orig_dir}"
+      popd || { echo "Unable to return to original directory after FTL binary download."; return 1; }
       # Install the FTL service
-      install -T -m 0755 "${PI_HOLE_LOCAL_REPO}/advanced/pihole-FTL.service" "/etc/init.d/pihole-FTL"
       echo -e "${OVER}  ${TICK} ${str}"
       return 0
     # Otherise,
     else
+      # the download failed, so just go back to the original directory
+      popd || { echo "Unable to return to original directory after FTL binary download."; return 1; }
       echo -e "${OVER}  ${CROSS} ${str}"
       echo -e "  ${COL_LIGHT_RED}Error: Download of binary from Github failed${COL_NC}"
-      # the download failed, so just go back to the original directory
-      cd "${orig_dir}"
       return 1
     fi
   # Otherwise,
   else
-    cd "${orig_dir}"
+    popd || { echo "Unable to return to original directory after FTL binary download."; return 1; }
     echo -e "${OVER}  ${CROSS} ${str}"
     # The URL could not be found
     echo -e "  ${COL_LIGHT_RED}Error: URL not found${COL_NC}"
+    return 1
   fi
 }
 
@@ -1957,7 +1962,7 @@ main() {
   for var in "$@"; do
     case "$var" in
       "--reconfigure" ) reconfigure=true;;
-      "--i_do_not_follow_recommendations" ) skipSpaceCheck=false;;
+      "--i_do_not_follow_recommendations" ) skipSpaceCheck=true;;
       "--unattended" ) runUnattended=true;;
     esac
   done

gravity.sh

@@ -11,6 +11,8 @@
 # This file is copyright under the latest version of the EUPL.
 # Please see LICENSE file for your rights under this license.
 
+export LC_ALL=C
+
 coltable="/opt/pihole/COL_TABLE"
 source "${coltable}"
 
@@ -417,24 +419,6 @@ gravity_SortAndFilterConsolidatedList() {
   echo -e "  ${INFO} Number of unique domains trapped in the Event Horizon: ${COL_BLUE}${num}${COL_NC}"
 }
 
-# Whitelist unique blocklist domain sources
-gravity_WhitelistBlocklistSourceUrls() {
-  local uniqDomains str
-
-  echo ""
-
-  # Create array of unique $sourceDomains
-  mapfile -t uniqDomains <<< "$(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")")"
-
-  str="Number of blocklist source domains being added to the whitelist: ${#uniqDomains[@]}"
-  echo -ne "  ${INFO} ${str}..."
-
-  # Whitelist $uniqDomains
-  "${PIHOLE_COMMAND}" -w -nr -q ${uniqDomains[*]} &> /dev/null
-
-  echo -e "${OVER}  ${INFO} ${str}"
-}
-
 # Whitelist user-defined domains
 gravity_Whitelist() {
   local num str
@@ -521,8 +505,13 @@ gravity_ParseBlacklistDomains() {
 
   # Empty $accretionDisc if it already exists, otherwise, create it
   : > "${piholeDir}/${accretionDisc}"
-
+  
-  gravity_ParseDomainsIntoHosts "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}"
+  if [[ -f "${piholeDir}/${whitelistMatter}" ]]; then
+    gravity_ParseDomainsIntoHosts "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}"
+  else
+    # There was no whitelist file, so use preEventHorizon instead of whitelistMatter.
+    gravity_ParseDomainsIntoHosts "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}"
+  fi
 
   # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it
   output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 )
@@ -630,7 +619,6 @@ if [[ "${skipDownload}" == false ]]; then
   gravity_SetDownloadOptions
   gravity_ConsolidateDownloadedBlocklists
   gravity_SortAndFilterConsolidatedList
-  gravity_WhitelistBlocklistSourceUrls
 else
   # Gravity needs to modify Blacklist/Whitelist/Wildcards
   echo -e "  ${INFO} Using cached Event Horizon list..."