Merge pull request #1943 from pi-hole/release/3.3

Pi-hole v3.3

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,32 +1,31 @@
-**By submitting this pull request, I confirm the following:** `{please fill any appropriate checkboxes, e.g: [X]}`
+**By submitting this pull request, I confirm the following:** 
+*please fill any appropriate checkboxes, e.g: [X]*
 
-`{Please ensure that your pull request is for the 'development' branch!}`
+- [ ] I have read and understood the [contributors guide](https://github.com/pi-hole/pi-hole/blob/master/CONTRIBUTING.md), as well as this entire template.
+- [ ] I have made only one major change in my proposed changes.
+- [ ] I have commented my proposed changes within the code.
+- [ ] I have tested my proposed changes, and have included unit tests where possible.
+- [ ] I am willing to help maintain this change if there are issues with it later.
+- [ ] I give this submission freely and claim no ownership.
+- [ ] It is compatible with the [EUPL 1.2 license](https://opensource.org/licenses/EUPL-1.1)
+- [ ] I have squashed any insignificant commits. ([`git rebase`](http://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html))
 
-- [] I have read and understood the [contributors guide](https://github.com/pi-hole/pi-hole/blob/master/CONTRIBUTING.md), as well as this entire template.
-- [] I have made only one major change in my proposed changes.
-- [] I have commented my proposed changes within the code.
-- [] I have tested my proposed changes, and have included unit tests where possible.
-- [] I am willing to help maintain this change if there are issues with it later.
-- [] I give this submission freely and claim no ownership.
-- [] It is compatible with the [EUPL 1.2 license](https://opensource.org/licenses/EUPL-1.1)
-- [] I have squashed any insignificant commits. ([`git rebase`](http://gitready.com/advanced/2009/02/10/squashing-commits-with-rebase.html))
-- [] I have Signed Off all commits. ([`git commit --signoff`](https://git-scm.com/docs/git-commit#git-commit---signoff))
+Please make sure you [Sign Off](https://github.com/pi-hole/pi-hole/wiki/How-to-signoff-your-commits.) all commits. Pi-hole enforces the [DCO](https://github.com/pi-hole/pi-hole/wiki/Contributing-to-the-project).
 
 ---
-
 **What does this PR aim to accomplish?:**
+*A detailed description, screenshots (if necessary), as well as links to any relevant GitHub issues*
 
-`{A detailed description, screenshots (if necessary), as well as links to any relevant GitHub issues}`
 
 **How does this PR accomplish the above?:**
+*A detailed description (such as a changelog) and screenshots (if necessary) of the implemented fix*
 
-`{A detailed description (such as a changelog) and screenshots (if necessary) of the implemented fix}`
 
 **What documentation changes (if any) are needed to support this PR?:**
+*A detailed list of any necessary changes*
 
-`{A detailed list of any necessary changes}`
 
-> * `{Please delete this quoted section when opening your pull request}`
-> * You must follow the template instructions. Failure to do so will result in your issue being closed.
-> * Please respect that Pi-hole is developed by volunteers, who can only reply in their spare time.
-> * Detail helps us understand an issue quicker, but please ensure it's relevant.
+---
+* You must follow the template instructions. Failure to do so will result in your pull request being closed.
+* Please respect that Pi-hole is developed by volunteers, who can only reply in their spare time.
+

.gitignore

@@ -3,3 +3,69 @@
 *.swp
 __pycache__
 .cache
+
+# Created by https://www.gitignore.io/api/jetbrains+iml
+
+### JetBrains+iml ###
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# All idea files, with execptions
+.idea
+!.idea/codeStyles/*
+!.idea/codeStyleSettings.xml
+
+
+# Sensitive or high-churn files:
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.xml
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+
+# Gradle:
+.idea/**/gradle.xml
+.idea/**/libraries
+
+# CMake
+cmake-build-debug/
+
+# Mongo Explorer plugin:
+.idea/**/mongoSettings.xml
+
+## File-based project format:
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Cursive Clojure plugin
+.idea/replstate.xml
+
+# Ruby plugin and RubyMine
+/.rakeTasks
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties
+
+### JetBrains+iml Patch ###
+# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023
+
+*.iml
+.idea/misc.xml
+*.ipr
+
+# End of https://www.gitignore.io/api/jetbrains+iml

.idea/codeStyles/Project.xml

@@ -0,0 +1,13 @@
+<component name="ProjectCodeStyleConfiguration">
+  <code_scheme name="Project" version="173">
+    <option name="OTHER_INDENT_OPTIONS">
+      <value>
+        <option name="INDENT_SIZE" value="2" />
+        <option name="TAB_SIZE" value="2" />
+      </value>
+    </option>
+    <MarkdownNavigatorCodeStyleSettings>
+      <option name="RIGHT_MARGIN" value="72" />
+    </MarkdownNavigatorCodeStyleSettings>
+  </code_scheme>
+</component>

.idea/codeStyles/codeStyleConfig.xml

@@ -0,0 +1,5 @@
+<component name="ProjectCodeStyleConfiguration">
+  <state>
+    <option name="USE_PER_PROJECT_SETTINGS" value="true" />
+  </state>
+</component>

CONTRIBUTING.md

@@ -28,6 +28,7 @@ When requesting or submitting new features, first consider whether it might be u
 
 - Check the codebase to ensure that your feature doesn't already exist.
 - Check the pull requests to ensure that another person hasn't already submitted the feature or fix.
+- Read and understand the [DCO guidelines](https://github.com/pi-hole/pi-hole/wiki/Contributing-to-the-project) for the project.
 
 ## Technical Requirements
 

README.md

@@ -185,7 +185,7 @@ While quite outdated at this point, [this original blog post about Pi-hole](http
 - [CHiP-hole: Network-wide Ad-blocker](https://www.hackster.io/jacobsalmela/chip-hole-network-wide-ad-blocker-98e037)
 - [Chrome Extension: Pi-Hole List Editor](https://chrome.google.com/webstore/detail/pi-hole-list-editor/hlnoeoejkllgkjbnnnhfolapllcnaglh) ([Source Code](https://github.com/packtloss/pihole-extension))
 - [Splunk: Pi-hole Visualiser](https://splunkbase.splunk.com/app/3023/)
-- [Adblocking with P-hole and Ubuntu 14.04 on VirtualBox](https://hbalagtas.blogspot.com.au/2016/02/adblocking-with-pi-hole-and-ubuntu-1404.html)
+- [Adblocking with Pi-hole and Ubuntu 14.04 on VirtualBox](https://hbalagtas.blogspot.com.au/2016/02/adblocking-with-pi-hole-and-ubuntu-1404.html)
 - [Pi-hole stats in your Mac's menu bar](https://getbitbar.com/plugins/Network/pi-hole.1m.py)
 - [Pi-hole unRAID Template](https://forums.lime-technology.com/topic/36810-support-spants-nodered-mqtt-dashing-couchdb/)
 - [Copernicus: Windows Tray Application](https://github.com/goldbattle/copernicus)
@@ -193,7 +193,7 @@ While quite outdated at this point, [this original blog post about Pi-hole](http
 - [Pi-hole metrics](https://github.com/nlamirault/pihole_exporter) exporter for [Prometheus](https://prometheus.io/)
 - [Magic Mirror with DNS Filtering](https://zonksec.com/blog/magic-mirror-dns-filtering/#dnssoftware)
 - [Pi-hole Droid: Android client](https://github.com/friimaind/pi-hole-droid)
-
+- [Windows DNS Swapper](https://github.com/roots84/DNS-Swapper), see [#1400](https://github.com/pi-hole/pi-hole/issues/1400)
 -----
 
 ## Coverage

advanced/01-pihole.conf

@@ -39,7 +39,7 @@ interface=@INT@
 
 cache-size=10000
 
-log-queries
+log-queries=extra
 log-facility=/var/log/pihole.log
 
 local-ttl=2

advanced/Scripts/chronometer.sh

@@ -15,7 +15,7 @@ pihole-FTL() {
   ftl_port=$(cat /var/run/pihole-FTL.port 2> /dev/null)
   if [[ -n "$ftl_port" ]]; then
     # Open connection to FTL
-    exec 3<>"/dev/tcp/localhost/$ftl_port"
+    exec 3<>"/dev/tcp/127.0.0.1/$ftl_port"
 
     # Test if connection is open
     if { "true" >&3; } 2> /dev/null; then

advanced/Scripts/piholeDebug.sh

@@ -215,14 +215,14 @@ copy_to_debug_log() {
   sed 's/\[[0-9;]\{1,5\}m//g' > "${PIHOLE_DEBUG_LOG_SANITIZED}" <<< cat "${PIHOLE_DEBUG_LOG}"
 }
 
-initiate_debug() {
+initialize_debug() {
   # Clear the screen so the debug log is readable
   clear
   show_disclaimer
   # Display that the debug process is beginning
   log_write "${COL_PURPLE}*** [ INITIALIZING ]${COL_NC}"
   # Timestamp the start of the log
-  log_write "${INFO} $(date "+%Y-%m-%d:%H:%M:%S") debug log has been initiated."
+  log_write "${INFO} $(date "+%Y-%m-%d:%H:%M:%S") debug log has been initialized."
 }
 
 # This is a function for visually displaying the curent test that is being run.
@@ -547,7 +547,7 @@ detect_ip_addresses() {
     log_write ""
   else
     # If there are no IPs detected, explain that the protocol is not configured
-    log_write "${CROSS} ${COL_RED}No IPv${protocol} address(es) found on the ${PIHOLE_INTERFACE}${COL_NC} interace.\n"
+    log_write "${CROSS} ${COL_RED}No IPv${protocol} address(es) found on the ${PIHOLE_INTERFACE}${COL_NC} interface.\n"
     return 1
   fi
   # If the protocol is v6
@@ -1149,7 +1149,7 @@ upload_to_tricorder() {
 
 # Run through all the functions we made
 make_temporary_log
-initiate_debug
+initialize_debug
 # setupVars.conf needs to be sourced before the networking so the values are
 # available to the other functions
 source_setup_variables

advanced/Scripts/updatecheck.sh

@@ -47,7 +47,7 @@ if [[ "$2" == "remote" ]]; then
   GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")"
   GITHUB_FTL_VERSION="$(json_extract tag_name "$(curl -q 'https://api.github.com/repos/pi-hole/FTL/releases/latest' 2> /dev/null)")"
 
-  echo "${GITHUB_CORE_VERSION} ${GITHUB_WEB_VERSION} ${GITHUB_FTL_VERSION}" > "/etc/pihole/GitHubVersions"
+  echo -n "${GITHUB_CORE_VERSION} ${GITHUB_WEB_VERSION} ${GITHUB_FTL_VERSION}" > "/etc/pihole/GitHubVersions"
 
 else
 
@@ -55,12 +55,12 @@ else
   WEB_BRANCH="$(get_local_branch /var/www/html/admin)"
   FTL_BRANCH="$(pihole-FTL branch)"
 
-  echo "${CORE_BRANCH} ${WEB_BRANCH} ${FTL_BRANCH}" > "/etc/pihole/localbranches"
+  echo -n "${CORE_BRANCH} ${WEB_BRANCH} ${FTL_BRANCH}" > "/etc/pihole/localbranches"
 
   CORE_VERSION="$(get_local_version /etc/.pihole)"
   WEB_VERSION="$(get_local_version /var/www/html/admin)"
   FTL_VERSION="$(pihole-FTL version)"
 
-  echo "${CORE_VERSION} ${WEB_VERSION} ${FTL_VERSION}" > "/etc/pihole/localversions"
+  echo -n "${CORE_VERSION} ${WEB_VERSION} ${FTL_VERSION}" > "/etc/pihole/localversions"
 
 fi

advanced/Scripts/webpage.sh

@@ -175,6 +175,11 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423
 		add_dnsmasq_setting "local-service"
 	else
 		# Listen only on one interface
+		# Use eth0 as fallback interface if interface is missing in setupVars.conf
+		if [ -z "${PIHOLE_INTERFACE}" ]; then
+			PIHOLE_INTERFACE="eth0"
+		fi
+
 		add_dnsmasq_setting "interface" "${PIHOLE_INTERFACE}"
 	fi
 
@@ -241,7 +246,7 @@ ProcessDHCPSettings() {
 	source "${setupVars}"
 
 	if [[ "${DHCP_ACTIVE}" == "true" ]]; then
-    interface=$(grep 'PIHOLE_INTERFACE=' /etc/pihole/setupVars.conf | sed "s/.*=//")
+    interface="${PIHOLE_INTERFACE}"
 
     # Use eth0 as fallback interface
     if [ -z ${interface} ]; then
@@ -249,7 +254,7 @@ ProcessDHCPSettings() {
     fi
 
     if [[ "${PIHOLE_DOMAIN}" == "" ]]; then
-      PIHOLE_DOMAIN="local"
+      PIHOLE_DOMAIN="lan"
       change_setting "PIHOLE_DOMAIN" "${PIHOLE_DOMAIN}"
     fi
 

advanced/pihole-FTL.service

@@ -26,6 +26,10 @@ start() {
     echo "pihole-FTL is already running"
   else
     touch /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole.log
+    mkdir -p /var/run/pihole
+    mkdir -p /var/log/pihole
+    chown pihole:pihole /var/run/pihole /var/log/pihole
+    rm /var/run/pihole/FTL.sock
     chown pihole:pihole /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /etc/pihole
     chmod 0644 /var/log/pihole-FTL.log /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole.log
     su -s /bin/sh -c "/usr/bin/pihole-FTL" "$FTLUSER"

advanced/pihole.cron

@@ -18,9 +18,6 @@
 #          early morning. Download any updates from the adlists
 59 1    * * 7   root    PATH="$PATH:/usr/local/bin/" pihole updateGravity
 
-# Pi-hole: Update Pi-hole! Uncomment to enable auto update
-#30 2    * * 7   root    PATH="$PATH:/usr/local/bin/" pihole updatePihole
-
 # Pi-hole: Flush the log daily at 00:00
 #          The flush script will use logrotate if available
 #          parameter "once": logrotate only once (default is twice)

automated install/basic-install.sh

@@ -2,7 +2,7 @@
 # shellcheck disable=SC1090
 
 # Pi-hole: A black hole for Internet advertisements
-# (c) 2017 Pi-hole, LLC (https://pi-hole.net)
+# (c) 2017-2018 Pi-hole, LLC (https://pi-hole.net)
 # Network-wide ad blocking via your own hardware.
 #
 # Installs and Updates Pi-hole
@@ -14,7 +14,7 @@
 #
 # Install with this command (from your Linux machine):
 #
-# curl -L install.pi-hole.net | bash
+# curl -sSL https://install.pi-hole.net | bash
 
 # -e option instructs bash to immediately exit if any command [1] has a non-zero exit status
 # We do not want users to end up with a partially working install, so we exit the script
@@ -28,9 +28,8 @@ set -e
 # Local variables will be in lowercase and will exist only within functions
 # It's still a work in progress, so you may see some variance in this guideline until it is complete
 
-# We write to a temporary file before moving the log to the pihole folder
-tmpLog=/tmp/pihole-install.log
-instalLogLoc=/etc/pihole/install.log
+# Location for final installation log storage
+installLogLoc=/etc/pihole/install.log
 # This is an important file as it contains information specific to the machine it's being installed on
 setupVars=/etc/pihole/setupVars.conf
 # Pi-hole uses lighttpd as a Web server, and this is the config file for it
@@ -210,7 +209,8 @@ elif command -v rpm &> /dev/null; then
   INSTALLER_DEPS=(dialog git iproute net-tools newt procps-ng)
   PIHOLE_DEPS=(bc bind-utils cronie curl dnsmasq findutils nmap-ncat sudo unzip wget libidn2 psmisc)
   PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php php-common php-cli php-pdo)
-  if ! grep -q 'Fedora' /etc/redhat-release; then
+  # EPEL (https://fedoraproject.org/wiki/EPEL) is required for lighttpd on CentOS
+  if grep -qi 'centos' /etc/redhat-release; then
     INSTALLER_DEPS=("${INSTALLER_DEPS[@]}" "epel-release");
   fi
     LIGHTTPD_USER="lighttpd"
@@ -815,6 +815,7 @@ setDNS() {
     Quad9)
       echo "Quad9 servers"
       PIHOLE_DNS_1="9.9.9.9"
+      PIHOLE_DNS_2="149.112.112.112"
       ;;
     Custom)
       # Until the DNS settings are selected,
@@ -1890,14 +1891,28 @@ FTLdetect() {
 	  # Install FTL
     FTLinstall "${binary}" || return 1
   fi
+}
 
+make_temporary_log() {
+  # Create a random temporary file for the log
+  TEMPLOG=$(mktemp /tmp/pihole_temp.XXXXXX)
+  # Open handle 3 for templog
+  # https://stackoverflow.com/questions/18460186/writing-outputs-to-log-file-and-console
+  exec 3>"$TEMPLOG"
+  # Delete templog, but allow for addressing via file handle
+  # This lets us write to the log without having a temporary file on the drive, which
+  # is meant to be a security measure so there is not a lingering file on the drive during the install process
+  rm "$TEMPLOG"
+}
 
+copy_to_install_log() {
+  # Copy the contents of file descriptor 3 into the install log
+  # Since we use color codes such as '\e[1;33m', they should be removed
+  sed 's/\[[0-9;]\{1,5\}m//g' < /proc/$$/fd/3 > "${installLogLoc}"
 }
 
 main() {
   ######## FIRST CHECK ########
-  # Show the Pi-hole logo so people know it's genuine since the logo and name are trademarked
-  show_ascii_berry
   # Must be root to install
   local str="Root user check"
   echo ""
@@ -1906,12 +1921,15 @@ main() {
   if [[ "${EUID}" -eq 0 ]]; then
     # they are root and all is good
     echo -e "  ${TICK} ${str}"
+    # Show the Pi-hole logo so people know it's genuine since the logo and name are trademarked
+    show_ascii_berry
+    make_temporary_log
   # Otherwise,
   else
     # They do not have enough privileges, so let the user know
     echo -e "  ${CROSS} ${str}
       ${COL_LIGHT_RED}Script called with non-root privileges${COL_NC}
-      The Pi-hole requires elevated privleges to install and run
+      The Pi-hole requires elevated privileges to install and run
       Please check the installer for any concerns regarding this requirement
       Make sure to download this script from a trusted source\\n"
     echo -ne "  ${INFO} Sudo utility check"
@@ -2030,14 +2048,14 @@ main() {
     fi
 
     # Install and log everything to a file
-    installPihole | tee ${tmpLog}
+    installPihole | tee -a /proc/$$/fd/3
   else
+    # Source ${setupVars} to use predefined user variables in the functions
+    source ${setupVars}
+
     # Clone/Update the repos
     clone_or_update_repos
 
-    # Source ${setupVars} for use in the rest of the functions
-    source ${setupVars}
-
     # Install packages used by the Pi-hole
     if [[ "${INSTALL_WEB}" == true ]]; then
       # Install the Web dependencies
@@ -2056,12 +2074,11 @@ main() {
       # Value will either be 1, if true, or 0
       LIGHTTPD_ENABLED=$(service lighttpd status | awk '/Loaded:/ {print $0}' | grep -c 'enabled' || true)
     fi
-
-    updatePihole | tee ${tmpLog}
+    updatePihole | tee -a /proc/$$/fd/3
   fi
 
-  # Move the log file into /etc/pihole for storage
-  mv ${tmpLog} ${instalLogLoc}
+  # Copy the temp log file into final log location for storage
+  copy_to_install_log
 
   if [[ "${INSTALL_WEB}" == true ]]; then
     # Add password to web UI if there is none
@@ -2101,6 +2118,7 @@ main() {
 
   # Force an update of the updatechecker
   . /opt/pihole/updatecheck.sh
+  . /opt/pihole/updatecheck.sh x remote
 
   #
   if [[ "${useUpdateVars}" == false ]]; then
@@ -2137,7 +2155,7 @@ main() {
   fi
 
   # Display where the log file is
-  echo -e "\\n  ${INFO} The install log is located at: /etc/pihole/install.log
+  echo -e "\\n  ${INFO} The install log is located at: ${installLogLoc}
   ${COL_LIGHT_GREEN}${INSTALL_TYPE} Complete! ${COL_NC}"
 
 }

gravity.sh

@@ -68,7 +68,7 @@ if [[ -r "${piholeDir}/pihole.conf" ]]; then
 fi
 
 # Determine if DNS resolution is available before proceeding
-gravity_DNSLookup() {
+gravity_CheckDNSResolutionAvailable() {
   local lookupDomain="pi.hole"
 
   # Determine if $localList does not exist
@@ -120,11 +120,11 @@ gravity_DNSLookup() {
   done
 
   # Try again
-  gravity_DNSLookup
+  gravity_CheckDNSResolutionAvailable
 }
 
 # Retrieve blocklist URLs and parse domains from adlists.list
-gravity_Collapse() {
+gravity_GetBlocklistUrls() {
   echo -e "  ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
 
   # Determine if adlists file needs handling
@@ -165,7 +165,7 @@ gravity_Collapse() {
 }
 
 # Define options for when retrieving blocklists
-gravity_Supernova() {
+gravity_SetDownloadOptions() {
   local url domain agent cmd_ext str
 
   echo ""
@@ -190,7 +190,7 @@ gravity_Supernova() {
 
     if [[ "${skipDownload}" == false ]]; then
       echo -e "  ${INFO} Target: ${domain} (${url##*/})"
-      gravity_Pull "${url}" "${cmd_ext}" "${agent}"
+      gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}"
       echo ""
     fi
   done
@@ -198,7 +198,7 @@ gravity_Supernova() {
 }
 
 # Download specified URL and perform checks on HTTP status and file content
-gravity_Pull() {
+gravity_DownloadBlocklistFromUrl() {
   local url="${1}" cmd_ext="${2}" agent="${3}" heisenbergCompensator="" patternBuffer str httpCode success=""
 
   # Create temp file to store content on disk instead of RAM
@@ -330,7 +330,7 @@ gravity_ParseFileIntoDomains() {
       }' "${source}" > "${destination}.exceptionsFile.tmp"
 
       # Remove exceptions
-      grep -F -x -v -f "${destination}.exceptionsFile.tmp" "${destination}" > "${source}"
+      comm -23 "${destination}" <(sort "${destination}.exceptionsFile.tmp") > "${source}"
       mv "${source}" "${destination}"
     fi
 
@@ -365,7 +365,7 @@ gravity_ParseFileIntoDomains() {
 }
 
 # Create (unfiltered) "Matter and Light" consolidated list
-gravity_Schwarzschild() {
+gravity_ConsolidateDownloadedBlocklists() {
   local str lastLine
 
   str="Consolidating blocklists"
@@ -393,7 +393,7 @@ gravity_Schwarzschild() {
 }
 
 # Parse consolidated list into (filtered, unique) domains-only format
-gravity_Filter() {
+gravity_SortAndFilterConsolidatedList() {
   local str num
 
   str="Extracting domains from blocklists"
@@ -418,7 +418,7 @@ gravity_Filter() {
 }
 
 # Whitelist unique blocklist domain sources
-gravity_WhitelistBLD() {
+gravity_WhitelistBlocklistSourceUrls() {
   local uniqDomains str
 
   echo ""
@@ -449,7 +449,7 @@ gravity_Whitelist() {
   echo -ne "  ${INFO} ${str}..."
 
   # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile
-  grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${whitelistMatter}"
+  comm -23 "${piholeDir}/${preEventHorizon}" <(sort "${whitelistFile}") > "${piholeDir}/${whitelistMatter}"
 
   echo -e "${OVER}  ${INFO} ${str}"
 }
@@ -563,7 +563,7 @@ gravity_Cleanup() {
   rm ${piholeDir}/*.tmp 2> /dev/null
   rm /tmp/*.phgpb 2> /dev/null
 
-  # Ensure this function only runs when gravity_Supernova() has completed
+  # Ensure this function only runs when gravity_SetDownloadOptions() has completed
   if [[ "${gravity_Blackbody:-}" == true ]]; then
     # Remove any unused .domains files
     for file in ${piholeDir}/*.${domainsExtension}; do
@@ -625,12 +625,12 @@ fi
 # Determine which functions to run
 if [[ "${skipDownload}" == false ]]; then
   # Gravity needs to download blocklists
-  gravity_DNSLookup
-  gravity_Collapse
-  gravity_Supernova
-  gravity_Schwarzschild
-  gravity_Filter
-  gravity_WhitelistBLD
+  gravity_CheckDNSResolutionAvailable
+  gravity_GetBlocklistUrls
+  gravity_SetDownloadOptions
+  gravity_ConsolidateDownloadedBlocklists
+  gravity_SortAndFilterConsolidatedList
+  gravity_WhitelistBlocklistSourceUrls
 else
   # Gravity needs to modify Blacklist/Whitelist/Wildcards
   echo -e "  ${INFO} Using cached Event Horizon list..."

pihole

@@ -444,13 +444,17 @@ Specify whether the Pi-hole log should be used
 
 Options:
   on                  Enable the Pi-hole log at /var/log/pihole.log
-  off                 Disable the Pi-hole log at /var/log/pihole.log"
+  off                 Disable and flush the Pi-hole log at /var/log/pihole.log
+  off noflush         Disable the Pi-hole log at /var/log/pihole.log"
     exit 0
   elif [[ "${1}" == "off" ]]; then
     # Disable logging
     sed -i 's/^log-queries/#log-queries/' /etc/dnsmasq.d/01-pihole.conf
     sed -i 's/^QUERY_LOGGING=true/QUERY_LOGGING=false/' /etc/pihole/setupVars.conf
+    if [[ "${2}" != "noflush" ]]; then
+      # Flush logs
       pihole -f
+    fi
     echo -e "  ${INFO} Disabling logging..."
     local str="Logging has been disabled!"
   elif [[ "${1}" == "on" ]]; then