From 251c9fee98b853ed7caef205f4cffcd50845423c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 14 Aug 2019 20:58:18 +0200 Subject: [PATCH 01/38] Simplify restarting code for "pihole restartdns". This fixes #2869. Signed-off-by: DL6ER --- pihole | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pihole b/pihole index 84a5623f..90309eee 100755 --- a/pihole +++ b/pihole @@ -98,20 +98,16 @@ versionFunc() { restartDNS() { local svcOption svc str output status - svcOption="${1:-}" + svcOption="${1:-restart}" - # Determine if we should reload or restart restart + # Determine if we should reload or restart if [[ "${svcOption}" =~ "reload" ]]; then - # Using SIGHUP will NOT re-read any *.conf files + # Reload has been requested + # Note: This will NOT re-read any *.conf files svc="killall -s SIGHUP ${resolver}" else - # Get PID of resolver to determine if it needs to start or restart - if pidof pihole-FTL &> /dev/null; then - svcOption="restart" - else - svcOption="start" - fi - svc="service ${resolver} ${svcOption}" + # A full restart has been requested + svc="service ${resolver} restart" fi # Print output to Terminal, but not to Web Admin From 71d5b4272640a335f12e023fc1784d446919aec0 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 12 Jul 2019 20:03:36 -0700 Subject: [PATCH 02/38] Remove the ZeusTracker blocklist from the defaults It is no longer served. Fixes #2843. Signed-off-by: Mcat12 --- automated install/basic-install.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 5cb4e5cc..fb84ac86 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1168,12 +1168,11 @@ chooseBlocklists() { mv "${adlistFile}" "${adlistFile}.old" fi # Let user select (or not) blocklists via a checklist - cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 7) + cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 6) # In an array, show the options available (all off by default): options=(StevenBlack "StevenBlack's Unified Hosts List" on MalwareDom "MalwareDomains" on Cameleon "Cameleon" on - ZeusTracker "ZeusTracker" on DisconTrack "Disconnect.me Tracking" on DisconAd "Disconnect.me Ads" on HostsFile "Hosts-file.net Ads" on) @@ -1195,7 +1194,6 @@ appendToListsFile() { StevenBlack ) echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";; MalwareDom ) echo "https://mirror1.malwaredomains.com/files/justdomains" >> "${adlistFile}";; Cameleon ) echo "http://sysctl.org/cameleon/hosts" >> "${adlistFile}";; - ZeusTracker ) echo "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist" >> "${adlistFile}";; DisconTrack ) echo "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" >> "${adlistFile}";; DisconAd ) echo "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt" >> "${adlistFile}";; HostsFile ) echo "https://hosts-file.net/ad_servers.txt" >> "${adlistFile}";; @@ -1213,7 +1211,6 @@ installDefaultBlocklists() { appendToListsFile StevenBlack appendToListsFile MalwareDom appendToListsFile Cameleon - appendToListsFile ZeusTracker appendToListsFile DisconTrack appendToListsFile DisconAd appendToListsFile HostsFile From 9f86fd0cb48cd5ed6d6e9d743d9db5fab822c6d8 Mon Sep 17 00:00:00 2001 From: Jeroen Baert <3607063+Forceflow@users.noreply.github.com> Date: Mon, 1 Jul 2019 03:42:02 +0200 Subject: [PATCH 03/38] Fix for 404 error when browsing to pi.hole (without /admin) Signed-off-by: Jeroen Baert <3607063+Forceflow@users.noreply.github.com> --- advanced/lighttpd.conf.debian | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian index 47bdbee0..2215bbdb 100644 --- a/advanced/lighttpd.conf.debian +++ b/advanced/lighttpd.conf.debian @@ -27,7 +27,7 @@ server.modules = ( ) server.document-root = "/var/www/html" -server.error-handler-404 = "pihole/index.php" +server.error-handler-404 = "/pihole/index.php" server.upload-dirs = ( "/var/cache/lighttpd/uploads" ) server.errorlog = "/var/log/lighttpd/error.log" server.pid-file = "/var/run/lighttpd.pid" From d3d45a877664a9f4adf400f26cd7e47de03142ed Mon Sep 17 00:00:00 2001 From: Jeroen Baert <3607063+Forceflow@users.noreply.github.com> Date: Wed, 3 Jul 2019 04:28:23 +0200 Subject: [PATCH 04/38] Fix 404 error when browsing to pi.hole (without /admin) (for fedora) Signed-off-by: Jeroen Baert <3607063+Forceflow@users.noreply.github.com> --- advanced/lighttpd.conf.fedora | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora index 27bc33cc..4232c90f 100644 --- a/advanced/lighttpd.conf.fedora +++ b/advanced/lighttpd.conf.fedora @@ -28,7 +28,7 @@ server.modules = ( ) server.document-root = "/var/www/html" -server.error-handler-404 = "pihole/index.php" +server.error-handler-404 = "/pihole/index.php" server.upload-dirs = ( "/var/cache/lighttpd/uploads" ) server.errorlog = "/var/log/lighttpd/error.log" server.pid-file = "/var/run/lighttpd.pid" From d793ef1ab86b9c8a0ccd5933cb178731d2ec691b Mon Sep 17 00:00:00 2001 From: bcambl Date: Sat, 6 Jul 2019 10:57:57 -0600 Subject: [PATCH 05/38] Merge conflict Fedora Dockerfile for tests pinned to 30 Signed-off-by: Dan Schaper --- test/fedora.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fedora.Dockerfile b/test/fedora.Dockerfile index c4834388..a06fc007 100644 --- a/test/fedora.Dockerfile +++ b/test/fedora.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:latest +FROM fedora:30 ENV GITDIR /etc/.pihole ENV SCRIPTDIR /opt/pihole From 10de7f649b62bc5342a5408c1b7cb08b47ebe302 Mon Sep 17 00:00:00 2001 From: bcambl Date: Sat, 6 Jul 2019 10:58:19 -0600 Subject: [PATCH 06/38] add chkconfig to INSTALLER_DEPS (CentOS/Fedora) chkconfig is a dependency of spawn-fcgi which is a dependency of lighttpd which is installed via PIHOLE_WEB_DEPS in phase 2 adding chkconfig to INSTALLER_DEPS to ensure /etc/init.d is present during the installation prompts (phase 1) Signed-off-by: bcambl --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index fb84ac86..a426eba1 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -283,7 +283,7 @@ elif is_command rpm ; then UPDATE_PKG_CACHE=":" PKG_INSTALL=(${PKG_MANAGER} install -y) PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" - INSTALLER_DEPS=(dialog git iproute newt procps-ng which) + INSTALLER_DEPS=(dialog git iproute newt procps-ng which chkconfig) PIHOLE_DEPS=(bind-utils cronie curl findutils nmap-ncat sudo unzip wget libidn2 psmisc sqlite libcap) PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo) LIGHTTPD_USER="lighttpd" From 97e11bd94e92c277f823d1d24339167532243e04 Mon Sep 17 00:00:00 2001 From: bcambl Date: Sat, 6 Jul 2019 11:06:08 -0600 Subject: [PATCH 07/38] ensure installation dependencies for FTL tests which rely on /etc/init.d Signed-off-by: bcambl --- test/test_automated_install.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 853048d1..be8fdcfa 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -481,6 +481,13 @@ def test_FTL_download_aarch64_no_errors(Pihole): ''' confirms only aarch64 package is downloaded for FTL engine ''' + # mock whiptail answers and ensure installer dependencies + mock_command('whiptail', {'*': ('', '0')}, Pihole) + Pihole.run(''' + source /opt/pihole/basic-install.sh + distro_check + install_dependent_packages ${INSTALLER_DEPS[@]} + ''') download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh binary="pihole-FTL-aarch64-linux-gnu" @@ -495,6 +502,13 @@ def test_FTL_download_unknown_fails_no_errors(Pihole): ''' confirms unknown binary is not downloaded for FTL engine ''' + # mock whiptail answers and ensure installer dependencies + mock_command('whiptail', {'*': ('', '0')}, Pihole) + Pihole.run(''' + source /opt/pihole/basic-install.sh + distro_check + install_dependent_packages ${INSTALLER_DEPS[@]} + ''') download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh binary="pihole-FTL-mips" @@ -512,6 +526,13 @@ def test_FTL_download_binary_unset_no_errors(Pihole): ''' confirms unset binary variable does not download FTL engine ''' + # mock whiptail answers and ensure installer dependencies + mock_command('whiptail', {'*': ('', '0')}, Pihole) + Pihole.run(''' + source /opt/pihole/basic-install.sh + distro_check + install_dependent_packages ${INSTALLER_DEPS[@]} + ''') download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh FTLinstall From ab1ea5a36642fc123a848527b021d2a01e1d9843 Mon Sep 17 00:00:00 2001 From: "B. Olausson" Date: Mon, 29 Jul 2019 18:34:00 +0200 Subject: [PATCH 08/38] This change fixes issue #145 "stty: standard input: Inappropriate ioctl for device ".It checks if a real terminal exist, if not it sets the screen size to a fixed value. This helps to avoid nasty and unnecessary logs when running "pihole -up" via e.g. cron. Signed-off-by: B. Olausson --- automated install/basic-install.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a426eba1..81aac2c6 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -84,8 +84,13 @@ if [ -z "${USER}" ]; then fi -# Find the rows and columns will default to 80x24 if it can not be detected -screen_size=$(stty size || printf '%d %d' 24 80) +# Check if we are running on a real terminal and find the rows and columns +# If there is no real terminal, we will default to 80x24 +if [ -t 0 ] ; then + screen_size=$(stty size) +else + screen_size="24 80" +fi # Set rows variable to contain first number printf -v rows '%d' "${screen_size%% *}" # Set columns variable to contain second number From c53be459c6414f9a0d1e390a5a9362064e3825d2 Mon Sep 17 00:00:00 2001 From: Andreas Date: Sun, 4 Aug 2019 21:21:08 +0200 Subject: [PATCH 09/38] quick fix for when dig also returns a CNAME Signed-off-by: Andreas --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index ff0343e9..6d1730ef 100755 --- a/gravity.sh +++ b/gravity.sh @@ -263,7 +263,7 @@ gravity_DownloadBlocklistFromUrl() { else printf -v port "%s" "${PIHOLE_DNS_1#*#}" fi - ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}") + ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then port=443; else port=80 From 35cf863f4b500f889af058d0ce54e514e6cc7551 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Fri, 31 May 2019 22:12:54 -0700 Subject: [PATCH 10/38] Create FUNDING.yml Signed-off-by: Dan Schaper --- .github/FUNDING.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..3a75dc12 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,4 @@ +# These are supported funding model platforms + +patreon: pihole +custom: https://pi-hole.net/donate From c9829dd3e4ee8d713d565f78a029eda7772335b0 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 28 Jun 2019 20:49:56 -0700 Subject: [PATCH 11/38] Fix pihole -up showing FTL update when network is down Fixes #1877 Signed-off-by: Mcat12 --- automated install/basic-install.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 81aac2c6..5f21d4ee 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2386,6 +2386,11 @@ FTLcheckUpdate() { local FTLlatesttag FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep 'Location' | awk -F '/' '{print $NF}' | tr -d '\r\n') + if [[ $? != 0 ]]; then + # There was an issue while retrieving the latest version + return 3 + fi + if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then return 0 else From 4e0ad520010a5d69b37fb6f8596de3b8f15c2954 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 28 Jun 2019 20:57:05 -0700 Subject: [PATCH 12/38] Fix ShellCheck issue by refactoring a bit Signed-off-by: Mcat12 --- automated install/basic-install.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 5f21d4ee..b2a1c20f 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2383,14 +2383,16 @@ FTLcheckUpdate() { if [[ ${ftlLoc} ]]; then local FTLversion FTLversion=$(/usr/bin/pihole-FTL tag) + local FTLreleaseData local FTLlatesttag - FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep 'Location' | awk -F '/' '{print $NF}' | tr -d '\r\n') - if [[ $? != 0 ]]; then + if ! FTLreleaseData=$(curl -sI https://github.com/pi-hole/FTL/releases/latest); then # There was an issue while retrieving the latest version return 3 fi + FTLlatesttag=$(grep 'Location' < "${FTLreleaseData}" | awk -F '/' '{print $NF}' | tr -d '\r\n') + if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then return 0 else From da398c3d9c2b7deee42a4ad8f3f10ec58a8643dc Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 19 Jul 2019 17:35:21 -0700 Subject: [PATCH 13/38] Print an error message if the FTL release metadata download fails Signed-off-by: Mcat12 --- automated install/basic-install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index b2a1c20f..47147ad7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2388,6 +2388,7 @@ FTLcheckUpdate() { if ! FTLreleaseData=$(curl -sI https://github.com/pi-hole/FTL/releases/latest); then # There was an issue while retrieving the latest version + printf " %b Failed to retrieve latest FTL release metadata" "${CROSS}" return 3 fi From 484f618685b5be0b4a54bcc177a1d57c3b528011 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 May 2019 15:26:27 +0200 Subject: [PATCH 14/38] Use last PID in case pidof returns multiple PIDs for pihole-FTL Signed-off-by: DL6ER --- advanced/Templates/pihole-FTL.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 9eb183ed..7c7e533e 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -13,7 +13,7 @@ FTLUSER=pihole PIDFILE=/var/run/pihole-FTL.pid get_pid() { - pidof "pihole-FTL" + pidof "pihole-FTL" | awk '{print $(NF)}' } is_running() { From e27f50b8e5f75e773338d044b62458a100cc5729 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 31 May 2019 09:18:12 +0200 Subject: [PATCH 15/38] Try to obtain PID from PIDFILE. If this fails (file does not exist or is empty), fall back to using pidof + awk Signed-off-by: DL6ER --- advanced/Templates/pihole-FTL.service | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 7c7e533e..8a4c7ce6 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -13,6 +13,13 @@ FTLUSER=pihole PIDFILE=/var/run/pihole-FTL.pid get_pid() { + # First, try to obtain PID from PIDFILE + if [ -s "${PIDFILE}" ]; then + cat "${PIDFILE}" + return + fi + + # If the PIDFILE is empty or not available, obtain the PID using pidof pidof "pihole-FTL" | awk '{print $(NF)}' } From 209555c42e970c5b7bdd407b13aa17a4a2fb3d84 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 17 Jun 2019 16:13:31 -0700 Subject: [PATCH 16/38] Fix uninstall causing 403 errors and not removing packages The 403 lighttpd errors were caused by removing the lighttpd config directory and not removing lighttpd itself. This caused a subsequent Pi-hole reinstall to not have all of the required lighttpd config files. The error while removing packages was caused by combining arguments into a string instead of listing each argument. Signed-off-by: Mcat12 --- automated install/uninstall.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index d0a6dcf0..52b4eeb7 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -80,7 +80,7 @@ removeAndPurge() { case ${yn} in [Yy]* ) echo -ne " ${INFO} Removing ${i}..."; - ${SUDO} "${PKG_REMOVE} ${i}" &> /dev/null; + ${SUDO} ${PKG_REMOVE} "${i}" &> /dev/null; echo -e "${OVER} ${INFO} Removed ${i}"; break;; [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; @@ -132,12 +132,15 @@ removeNoPurge() { fi if package_check lighttpd > /dev/null; then - ${SUDO} rm -rf /etc/lighttpd/ &> /dev/null - echo -e " ${TICK} Removed lighttpd" - else - if [ -f /etc/lighttpd/lighttpd.conf.orig ]; then + if [[ -f /etc/lighttpd/lighttpd.conf.orig ]]; then ${SUDO} mv /etc/lighttpd/lighttpd.conf.orig /etc/lighttpd/lighttpd.conf fi + + if [[ -f /etc/lighttpd/external.conf ]]; then + ${SUDO} rm /etc/lighttpd/external.conf + fi + + echo -e " ${TICK} Removed lighttpd configs" fi ${SUDO} rm -f /etc/dnsmasq.d/adList.conf &> /dev/null From 25941647725bf170c8f4bb7d5da575ea0ad036aa Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 17 Jun 2019 16:28:55 -0700 Subject: [PATCH 17/38] Use an array for PKG_REMOVE Fixes shellcheck warning. Signed-off-by: Mcat12 --- automated install/uninstall.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 52b4eeb7..59880136 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -55,13 +55,13 @@ fi # Compatability if [ -x "$(command -v apt-get)" ]; then # Debian Family - PKG_REMOVE="${PKG_MANAGER} -y remove --purge" + PKG_REMOVE=("${PKG_MANAGER}" -y remove --purge) package_check() { dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" } elif [ -x "$(command -v rpm)" ]; then # Fedora Family - PKG_REMOVE="${PKG_MANAGER} remove -y" + PKG_REMOVE=("${PKG_MANAGER}" remove -y) package_check() { rpm -qa | grep "^$1-" > /dev/null } @@ -80,7 +80,7 @@ removeAndPurge() { case ${yn} in [Yy]* ) echo -ne " ${INFO} Removing ${i}..."; - ${SUDO} ${PKG_REMOVE} "${i}" &> /dev/null; + ${SUDO} "${PKG_REMOVE}" "${i}" &> /dev/null; echo -e "${OVER} ${INFO} Removed ${i}"; break;; [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; From d5d1a607adf9bf54a3023745c64e2d8da8c01be5 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 17 Jun 2019 16:30:26 -0700 Subject: [PATCH 18/38] Fix PKG_REMOVE array usage Signed-off-by: Mcat12 --- automated install/uninstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 59880136..5cd5b924 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -80,7 +80,7 @@ removeAndPurge() { case ${yn} in [Yy]* ) echo -ne " ${INFO} Removing ${i}..."; - ${SUDO} "${PKG_REMOVE}" "${i}" &> /dev/null; + ${SUDO} "${PKG_REMOVE[@]}" "${i}" &> /dev/null; echo -e "${OVER} ${INFO} Removed ${i}"; break;; [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; From b107ae2ab9977b0c56f240fb8f630d42e0ea1113 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Wed, 12 Jun 2019 19:13:29 -0700 Subject: [PATCH 19/38] Use the filtered IPv6 OpenDNS servers The ones we were using previously were not filtered. See https://support.opendns.com/hc/en-us/articles/227986667-Does-OpenDNS-Support-IPv6- Signed-off-by: Mcat12 --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 47147ad7..60278dfa 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -31,7 +31,7 @@ set -e # List of supported DNS servers DNS_SERVERS=$(cat << EOM Google (ECS);8.8.8.8;8.8.4.4;2001:4860:4860:0:0:0:0:8888;2001:4860:4860:0:0:0:0:8844 -OpenDNS (ECS);208.67.222.222;208.67.220.220;2620:0:ccc::2;2620:0:ccd::2 +OpenDNS (ECS);208.67.222.222;208.67.220.220;2620:119:35::35;2620:119:53::53 Level3;4.2.2.1;4.2.2.2;; Comodo;8.26.56.26;8.20.247.20;; DNS.WATCH;84.200.69.80;84.200.70.40;2001:1608:10:25:0:0:1c04:b12f;2001:1608:10:25:0:0:9249:d69b From 352146ef92cc54b31efa31f0a2d433f458558370 Mon Sep 17 00:00:00 2001 From: Andreas Kurth Date: Sun, 30 Jun 2019 08:02:51 +0200 Subject: [PATCH 20/38] Fix pihole manpage to match code. The dry-run argument to pihole -up is "--check-only", not "--checkonly". Signed-off-by: Andreas Kurth --- manpages/pihole.8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manpages/pihole.8 b/manpages/pihole.8 index bd7d0933..b3008563 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -35,7 +35,7 @@ pihole -g\fR .br \fBpihole\fR \fB-l\fR (\fBon|off|off noflush\fR) .br -\fBpihole -up \fR[--checkonly] +\fBpihole -up \fR[--check-only] .br \fBpihole -v\fR [-p|-a|-f] [-c|-l|-hash] .br From 34727c00c6c3a773db31a96a1478a6c4b291e02d Mon Sep 17 00:00:00 2001 From: David Haguenauer Date: Fri, 10 May 2019 10:37:38 -0400 Subject: [PATCH 21/38] Drop indirection from install_dependent_packages Previously, install_dependent_packages would receive an array variable name as its single parameter, and would use variable indirection to access it; this change simplifies that function so that it instead receives the expanded array. Signed-off-by: David Haguenauer --- automated install/basic-install.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 60278dfa..a67fff9c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1607,7 +1607,6 @@ install_dependent_packages() { # Install packages passed in via argument array # No spinner - conflicts with set -e - declare -a argArray1=("${!1}") declare -a installArray # Debian based package install - debconf will download the entire package list @@ -1617,7 +1616,7 @@ install_dependent_packages() { # installed by us, and remove only the installed packages, and not the entire list. if is_command debconf-apt-progress ; then # For each package, - for i in "${argArray1[@]}"; do + for i in "$@"; do printf " %b Checking for %s..." "${INFO}" "${i}" if dpkg-query -W -f='${Status}' "${i}" 2>/dev/null | grep "ok installed" &> /dev/null; then printf "%b %b Checking for %s\\n" "${OVER}" "${TICK}" "${i}" @@ -1636,7 +1635,7 @@ install_dependent_packages() { fi # Install Fedora/CentOS packages - for i in "${argArray1[@]}"; do + for i in "$@"; do printf " %b Checking for %s..." "${INFO}" "${i}" if ${PKG_MANAGER} -q list installed "${i}" &> /dev/null; then printf "%b %b Checking for %s" "${OVER}" "${TICK}" "${i}" @@ -2516,7 +2515,7 @@ main() { notify_package_updates_available # Install packages used by this installation script - install_dependent_packages INSTALLER_DEPS[@] + install_dependent_packages "${INSTALLER_DEPS[@]}" # Check if SELinux is Enforcing checkSelinux @@ -2567,7 +2566,7 @@ main() { dep_install_list+=("${PIHOLE_WEB_DEPS[@]}") fi - install_dependent_packages dep_install_list[@] + install_dependent_packages "${dep_install_list[@]}" unset dep_install_list # On some systems, lighttpd is not enabled on first install. We need to enable it here if the user From febdbceab1b0250fd86f6a8b9b9748d1e6b4a7dd Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Wed, 21 Aug 2019 10:03:54 -0400 Subject: [PATCH 22/38] Fix error when getting latest FTL tag The headers containing the latest FTL tag were not properly input to the command (`<` vs `<<<`). This caused Bash to try and open the file named after the header string, which does not exist. Signed-off-by: Mcat12 --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a67fff9c..6af88a57 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2391,7 +2391,7 @@ FTLcheckUpdate() { return 3 fi - FTLlatesttag=$(grep 'Location' < "${FTLreleaseData}" | awk -F '/' '{print $NF}' | tr -d '\r\n') + FTLlatesttag=$(grep 'Location' <<< "${FTLreleaseData}" | awk -F '/' '{print $NF}' | tr -d '\r\n') if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then return 0 From 03c65dd0e951343c68335eca5959501482fe35e5 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Sat, 24 Aug 2019 04:33:32 -0700 Subject: [PATCH 23/38] Convert hardcoded /usr/local/bin to variable Update pihole script with full path to 'pihole' Variable for webpage.sh 'pihole' call. Signed-off-by: Dan Schaper --- advanced/Scripts/webpage.sh | 3 ++- automated install/basic-install.sh | 11 ++++++----- pihole | 9 +++++---- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index c8dfb129..65b6587e 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -16,6 +16,7 @@ readonly dhcpconfig="/etc/dnsmasq.d/02-pihole-dhcp.conf" readonly FTLconf="/etc/pihole/pihole-FTL.conf" # 03 -> wildcards readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf" +readonly PI_HOLE_BIN_DIR="/usr/local/bin" coltable="/opt/pihole/COL_TABLE" if [[ -f ${coltable} ]]; then @@ -274,7 +275,7 @@ Reboot() { } RestartDNS() { - /usr/local/bin/pihole restartdns + "${PI_HOLE_BIN_DIR}"/pihole restartdns } SetQueryLogOptions() { diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 6af88a57..afd6ce23 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -65,6 +65,7 @@ PI_HOLE_FILES=(chronometer list piholeDebug piholeLogFlush setupLCD update versi # This directory is where the Pi-hole scripts will be installed PI_HOLE_INSTALL_DIR="/opt/pihole" PI_HOLE_CONFIG_DIR="/etc/pihole" +PI_HOLE_BIN_DIR="/usr/local/bin" PI_HOLE_BLOCKPAGE_DIR="${webroot}/pihole" useUpdateVars=false @@ -1339,7 +1340,7 @@ installScripts() { install -o "${USER}" -Dm755 -t "${PI_HOLE_INSTALL_DIR}" ./advanced/Scripts/*.sh install -o "${USER}" -Dm755 -t "${PI_HOLE_INSTALL_DIR}" ./automated\ install/uninstall.sh install -o "${USER}" -Dm755 -t "${PI_HOLE_INSTALL_DIR}" ./advanced/Scripts/COL_TABLE - install -o "${USER}" -Dm755 -t /usr/local/bin/ pihole + install -o "${USER}" -Dm755 -t "${PI_HOLE_BIN_DIR}" pihole install -Dm644 ./advanced/bash-completion/pihole /etc/bash_completion.d/pihole printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" @@ -1692,13 +1693,13 @@ installPiholeWeb() { # and copy in the pihole sudoers file install -m 0640 ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.sudo /etc/sudoers.d/pihole # Add lighttpd user (OS dependent) to sudoers file - echo "${LIGHTTPD_USER} ALL=NOPASSWD: /usr/local/bin/pihole" >> /etc/sudoers.d/pihole + echo "${LIGHTTPD_USER} ALL=NOPASSWD: ${PI_HOLE_BIN_DIR}/pihole" >> /etc/sudoers.d/pihole # If the Web server user is lighttpd, if [[ "$LIGHTTPD_USER" == "lighttpd" ]]; then # Allow executing pihole via sudo with Fedora - # Usually /usr/local/bin is not permitted as directory for sudoable programs - echo "Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin" >> /etc/sudoers.d/pihole + # Usually /usr/local/bin ${PI_HOLE_BIN_DIR} is not permitted as directory for sudoable programs + echo "Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:${PI_HOLE_BIN_DIR}" >> /etc/sudoers.d/pihole fi # Set the strict permissions on the file chmod 0440 /etc/sudoers.d/pihole @@ -2674,7 +2675,7 @@ main() { if [[ "${INSTALL_TYPE}" == "Update" ]]; then printf "\\n" - /usr/local/bin/pihole version --current + "${PI_HOLE_BIN_DIR}" version --current fi } diff --git a/pihole b/pihole index 90309eee..5befe926 100755 --- a/pihole +++ b/pihole @@ -10,6 +10,7 @@ # Please see LICENSE file for your rights under this license. readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" +readonly PI_HOLE_BIN_DIR="/usr/local/bin" readonly gravitylist="/etc/pihole/gravity.list" readonly blacklist="/etc/pihole/black.list" @@ -160,7 +161,7 @@ Time: local str="Disabling blocking for ${tt} seconds" echo -e " ${INFO} ${str}..." local str="Blocking will be re-enabled in ${tt} seconds" - nohup bash -c "sleep ${tt}; pihole enable" /dev/null & + nohup bash -c "sleep ${tt}; ${PI_HOLE_BIN_DIR}/pihole enable" /dev/null & else local error=true fi @@ -171,7 +172,7 @@ Time: echo -e " ${INFO} ${str}..." local str="Blocking will be re-enabled in ${tt} minutes" tt=$((${tt}*60)) - nohup bash -c "sleep ${tt}; pihole enable" /dev/null & + nohup bash -c "sleep ${tt}; ${PI_HOLE_BIN_DIR}/pihole enable" /dev/null & else local error=true fi @@ -233,7 +234,7 @@ Options: sed -i 's/^QUERY_LOGGING=true/QUERY_LOGGING=false/' /etc/pihole/setupVars.conf if [[ "${2}" != "noflush" ]]; then # Flush logs - pihole -f + "${PI_HOLE_BIN_DIR}"/pihole -f fi echo -e " ${INFO} Disabling logging..." local str="Logging has been disabled!" @@ -286,7 +287,7 @@ statusFunc() { *) echo -e " ${INFO} Pi-hole blocking will be enabled";; esac # Enable blocking - pihole enable + "${PI_HOLE_BIN_DIR}"/pihole enable fi } From 4cfe463dfaf2be13dc499e3b9b9dab79e45b56a1 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Sat, 24 Aug 2019 04:57:23 -0700 Subject: [PATCH 24/38] Add back dropped binary call. Signed-off-by: Dan Schaper --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index afd6ce23..f99d02ab 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2675,7 +2675,7 @@ main() { if [[ "${INSTALL_TYPE}" == "Update" ]]; then printf "\\n" - "${PI_HOLE_BIN_DIR}" version --current + "${PI_HOLE_BIN_DIR}"/pihole version --current fi } From 989d1aff60f92f10763bfe273d5ceddeaeab631b Mon Sep 17 00:00:00 2001 From: Niklas Elmose Andersen Date: Mon, 26 Aug 2019 19:05:14 +0200 Subject: [PATCH 25/38] Restore and improve 'pihole -q' matching Removes regex lookaround which 'grep -E' does not support. Restores support for blocklists in hosts format. Simplifies domain match cleanup logic by eliminating an if-condition. Improves domain matching by eliminating commented domain names, eliminating false positives in a few edge cases. Signed-off-by: Niklas Elmose Andersen --- advanced/Scripts/query.sh | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 69a3c7a4..123f8ea6 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -54,7 +54,7 @@ scanList(){ # /dev/null forces filename to be printed when only one list has been generated # shellcheck disable=SC2086 case "${type}" in - "exact" ) grep -i -E -l "(^|(?/dev/null;; + "exact" ) grep -i -E "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;; "wc" ) grep -i -o -m 1 "/${domain}/" ${lists} 2>/dev/null;; * ) grep -i "${domain}" ${lists} /dev/null 2>/dev/null;; esac @@ -170,19 +170,20 @@ elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then exit 0 fi -# Remove unwanted content from non-exact $results -if [[ -z "${exact}" ]]; then - # Delete lines starting with # - # Remove comments after domain - # Remove hosts format IP address - mapfile -t results <<< "$(IFS=$'\n'; sed \ - -e "/:#/d" \ - -e "s/[ \\t]#.*//g" \ - -e "s/:.*[ \\t]/:/g" \ - <<< "${results[*]}")" - # Exit if result was in a comment - [[ -z "${results[*]}" ]] && exit 0 -fi +# Remove unwanted content from $results +# Each line in $results is formatted as such: [fileName]:[line] +# 1. Delete lines starting with # +# 2. Remove comments after domain +# 3. Remove hosts format IP address +# 4. Remove any lines that no longer contain the queried domain name (in case the matched domain name was in a comment) +mapfile -t results <<< "$(IFS=$'\n'; sed \ + -e "/:#/d" \ + -e "s/[ \\t]#.*//g" \ + -e "s/:.*[ \\t]/:/g" \ + -e "/${domainQuery}/!d" \ + <<< "${results[*]}")" +# Exit if result was in a comment +[[ -z "${results[*]}" ]] && exit 0 # Get adlist file content as array if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then From a3e32d9a15373488320a48ced2c9fc5990bc51dd Mon Sep 17 00:00:00 2001 From: Niklas Elmose Andersen Date: Tue, 27 Aug 2019 12:13:28 +0200 Subject: [PATCH 26/38] Properly escape domain regex Dots in domain names should not match any character. Signed-off-by: Niklas Elmose Andersen --- advanced/Scripts/query.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 123f8ea6..83bf371c 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -176,11 +176,12 @@ fi # 2. Remove comments after domain # 3. Remove hosts format IP address # 4. Remove any lines that no longer contain the queried domain name (in case the matched domain name was in a comment) +esc_domain="${domainQuery//./\\.}" mapfile -t results <<< "$(IFS=$'\n'; sed \ -e "/:#/d" \ -e "s/[ \\t]#.*//g" \ -e "s/:.*[ \\t]/:/g" \ - -e "/${domainQuery}/!d" \ + -e "/${esc_domain}/!d" \ <<< "${results[*]}")" # Exit if result was in a comment [[ -z "${results[*]}" ]] && exit 0 From d8eee47ca4bfd2c250ba24fd6111b0276fa256c4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 May 2019 16:41:37 +0200 Subject: [PATCH 27/38] Add dhcp-ignore-names option when enabling DHCP service. We currently remove anything that starts with "dhcp-" to have a clean configuration and removed these lines without noticing when enabling the DHCP server. Signed-off-by: DL6ER --- advanced/01-pihole.conf | 5 ----- advanced/Scripts/webpage.sh | 8 ++++++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf index 502293bf..38d2c0b5 100644 --- a/advanced/01-pihole.conf +++ b/advanced/01-pihole.conf @@ -43,8 +43,3 @@ log-facility=/var/log/pihole.log local-ttl=2 log-async - -# If a DHCP client claims that its name is "wpad", ignore that. -# This fixes a security hole. see CERT Vulnerability VU#598349 -dhcp-name-match=set:wpad-ignore,wpad -dhcp-ignore-names=tag:wpad-ignore diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 65b6587e..600a45a5 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -364,6 +364,14 @@ EnableDHCP() { delete_dnsmasq_setting "dhcp-" delete_dnsmasq_setting "quiet-dhcp" + # If a DHCP client claims that its name is "wpad", ignore that. + # This fixes a security hole. see CERT Vulnerability VU#598349 + # We also ignore "localhost" as Windows behaves strangely if a + # device claims this host name + add_dnsmasq_setting "dhcp-name-match=set:hostname-ignore,wpad +dhcp-name-match=set:hostname-ignore,localhost +dhcp-ignore-names=tag:hostname-ignore" + ProcessDHCPSettings RestartDNS From 79b8dac0fa7b93e4082444a98f94bc920805b319 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 30 Aug 2019 22:06:14 +0100 Subject: [PATCH 28/38] Remove readonly attribute of the PI_HOLE_BIN_DIR declaration in pihole Signed-off-by: Adam Warner --- pihole | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pihole b/pihole index 5befe926..d2ff3645 100755 --- a/pihole +++ b/pihole @@ -10,14 +10,14 @@ # Please see LICENSE file for your rights under this license. readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" -readonly PI_HOLE_BIN_DIR="/usr/local/bin" readonly gravitylist="/etc/pihole/gravity.list" readonly blacklist="/etc/pihole/black.list" -# setupVars is not readonly here because in some funcitons (checkout), +# setupVars and PI_HOLE_BIN_DIR are not readonly here because in some funcitons (checkout), # it might get set again when the installer is sourced. This causes an # error due to modifying a readonly variable. setupVars="/etc/pihole/setupVars.conf" +PI_HOLE_BIN_DIR="/usr/local/bin" readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" source "${colfile}" From aed2e35bc0a26c8bebbbc06e0bf70bf6140f7f3c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 2 Sep 2019 22:39:28 +0200 Subject: [PATCH 29/38] Print messages only after removing possible matches in comments. Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 83bf371c..2dead97d 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -157,19 +157,6 @@ lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)") # Query blocklists for occurences of domain mapfile -t results <<< "$(scanList "${domainQuery}" "${lists[*]}" "${exact}")" -# Handle notices -if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then - echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the block lists" - exit 0 -elif [[ -z "${results[*]}" ]]; then - # Result found in WL/BL/Wildcards - exit 0 -elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then - echo -e " ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} - This can be overridden using the -all option" - exit 0 -fi - # Remove unwanted content from $results # Each line in $results is formatted as such: [fileName]:[line] # 1. Delete lines starting with # @@ -183,8 +170,19 @@ mapfile -t results <<< "$(IFS=$'\n'; sed \ -e "s/:.*[ \\t]/:/g" \ -e "/${esc_domain}/!d" \ <<< "${results[*]}")" -# Exit if result was in a comment -[[ -z "${results[*]}" ]] && exit 0 + +# Handle notices +if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then + echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the block lists" + exit 0 +elif [[ -z "${results[*]}" ]]; then + # Result found in WL/BL/Wildcards + exit 0 +elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then + echo -e " ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} + This can be overridden using the -all option" + exit 0 +fi # Get adlist file content as array if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then From ad41bcca5a415a42c99e5ccde9b74d7e8af206f0 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 3 Sep 2019 23:43:11 +0100 Subject: [PATCH 30/38] Remove support for adblock style lists to prevent false positives Signed-off-by: Adam Warner --- gravity.sh | 41 +---------------------------------------- 1 file changed, 1 insertion(+), 40 deletions(-) diff --git a/gravity.sh b/gravity.sh index 6d1730ef..6403fb9c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -353,46 +353,7 @@ gravity_ParseFileIntoDomains() { # Determine how to parse individual source file formats if [[ "${firstLine,,}" =~ (adblock|ublock|^!) ]]; then # Compare $firstLine against lower case words found in Adblock lists - echo -ne " ${INFO} Format: Adblock" - - # Define symbols used as comments: [! - # "||.*^" includes the "Example 2" domains we can extract - # https://adblockplus.org/filter-cheatsheet - abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/" - - # Parse Adblock lists by extracting "Example 2" domains - # Logic: Ignore lines which do not include comments or domain name anchor - awk ''"${abpFilter}"' { - # Remove valid adblock type options - gsub(/\$?~?(important|third-party|popup|subdocument|websocket),?/, "", $0) - # Remove starting domain name anchor "||" and ending seperator "^" - gsub(/^(\|\|)|(\^)/, "", $0) - # Remove invalid characters (*/,=$) - if($0 ~ /[*\/,=\$]/) { $0="" } - # Remove lines which are only IPv4 addresses - if($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } - if($0) { print $0 } - }' "${source}" > "${destination}" - - # Determine if there are Adblock exception rules - # https://adblockplus.org/filters - if grep -q "^@@||" "${source}" &> /dev/null; then - # Parse Adblock lists by extracting exception rules - # Logic: Ignore lines which do not include exception format "@@||example.com^" - awk -F "[|^]" '/^@@\|\|.*\^/ { - # Remove valid adblock type options - gsub(/\$?~?(third-party)/, "", $0) - # Remove invalid characters (*/,=$) - if($0 ~ /[*\/,=\$]/) { $0="" } - if($3) { print $3 } - }' "${source}" > "${destination}.exceptionsFile.tmp" - - # Remove exceptions - comm -23 "${destination}" <(sort "${destination}.exceptionsFile.tmp") > "${source}" - mv "${source}" "${destination}" - fi - - echo -e "${OVER} ${TICK} Format: Adblock" + echo -ne " ${CROSS} Format: Adblock (list type not supported)" elif grep -q "^address=/" "${source}" &> /dev/null; then # Parse Dnsmasq format lists echo -e " ${CROSS} Format: Dnsmasq (list type not supported)" From 8bef5dc805a6cfd530b96677f8439ffdcd96a12b Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 3 Sep 2019 23:52:35 +0100 Subject: [PATCH 31/38] remove n from -ne Signed-off-by: Adam Warner --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 6403fb9c..cca6279d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -353,7 +353,7 @@ gravity_ParseFileIntoDomains() { # Determine how to parse individual source file formats if [[ "${firstLine,,}" =~ (adblock|ublock|^!) ]]; then # Compare $firstLine against lower case words found in Adblock lists - echo -ne " ${CROSS} Format: Adblock (list type not supported)" + echo -e " ${CROSS} Format: Adblock (list type not supported)" elif grep -q "^address=/" "${source}" &> /dev/null; then # Parse Dnsmasq format lists echo -e " ${CROSS} Format: Dnsmasq (list type not supported)" From 525ec8cd01cfc069f4c670ac6e9c0aef0a17c02c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 7 Sep 2019 08:44:03 +0200 Subject: [PATCH 32/38] Signal to Firefox that the local network is unsuitable for DNS-over-HTTPS Signed-off-by: DL6ER --- advanced/01-pihole.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf index cd74e186..8aa35fe1 100644 --- a/advanced/01-pihole.conf +++ b/advanced/01-pihole.conf @@ -41,3 +41,8 @@ log-facility=/var/log/pihole.log local-ttl=2 log-async + +# Signal to Firefox that the local network is unsuitable for DNS-over-HTTPS +# This follows https://support.mozilla.org/en-US/kb/configuring-networks-disable-dns-over-https +# (sourced 7th September 2019) +server=/use-application-dns.net/ From 1f36ec48e3af54d849a191f48bdb6559158ce5f8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 7 Sep 2019 23:11:20 +0200 Subject: [PATCH 33/38] Add use-application-dns.net = NXDOMAIN in ProcessDNSSettings rather than in the template so we can ensure that it will survive config-renewals. Signed-off-by: DL6ER --- advanced/01-pihole.conf | 5 ----- advanced/Scripts/webpage.sh | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf index 8aa35fe1..cd74e186 100644 --- a/advanced/01-pihole.conf +++ b/advanced/01-pihole.conf @@ -41,8 +41,3 @@ log-facility=/var/log/pihole.log local-ttl=2 log-async - -# Signal to Firefox that the local network is unsuitable for DNS-over-HTTPS -# This follows https://support.mozilla.org/en-US/kb/configuring-networks-disable-dns-over-https -# (sourced 7th September 2019) -server=/use-application-dns.net/ diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 39299960..356c20ef 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -212,6 +212,11 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_DOMAIN}/${CONDITIONAL_FORWARDING_IP}" add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_REVERSE}/${CONDITIONAL_FORWARDING_IP}" fi + + # Prevent Firefox from automatically switching over to DNS-over-HTTPS + # This follows https://support.mozilla.org/en-US/kb/configuring-networks-disable-dns-over-https + # (sourced 7th September 2019) + add_dnsmasq_setting "server=/use-application-dns.net/" } SetDNSServers() { From f582344b9ac6698a5e500b20739994a6fed27cab Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 17 Sep 2019 21:59:48 +0200 Subject: [PATCH 34/38] "No default index.lighttpd.html file found... not backing up" is not an error. Signed-off-by: DL6ER --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 25c66ab7..e9684254 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1692,7 +1692,7 @@ installPiholeWeb() { # Otherwise, else # don't do anything - printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" + printf "%b %b %s\\n" "${OVER}" "${INFO}" "${str}" printf " No default index.lighttpd.html file found... not backing up\\n" fi From 9a6deb5a1a4c1d811c5a906d107e06d6973e7fcb Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 17 Sep 2019 21:16:49 +0100 Subject: [PATCH 35/38] Fix tests Signed-off-by: Adam Warner --- test/test_automated_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index cce11857..e8a4dede 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -338,7 +338,7 @@ def test_installPiholeWeb_fresh_install_no_errors(Pihole): expected_stdout = tick_box + (' Creating directory for blocking page, ' 'and copying files') assert expected_stdout in installWeb.stdout - expected_stdout = cross_box + ' Backing up index.lighttpd.html' + expected_stdout = info_box + ' Backing up index.lighttpd.html' assert expected_stdout in installWeb.stdout expected_stdout = ('No default index.lighttpd.html file found... ' 'not backing up') From 3cb4f6d9d4980c66a025f0e038acee9c3f3a20c4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 26 Sep 2019 13:50:54 +0200 Subject: [PATCH 36/38] We cannot create vw_gravity before having created vw_whitelist as the former depends onthe later. This commit changes the order in which the tables are created. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 09d581f0..d0c744f4 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -89,10 +89,6 @@ CREATE TABLE info INSERT INTO info VALUES("version","1"); -CREATE VIEW vw_gravity AS SELECT domain - FROM gravity - WHERE domain NOT IN (SELECT domain from vw_whitelist); - CREATE VIEW vw_whitelist AS SELECT DISTINCT domain FROM whitelist LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id @@ -141,3 +137,6 @@ CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; END; +CREATE VIEW vw_gravity AS SELECT domain + FROM gravity + WHERE domain NOT IN (SELECT domain from vw_whitelist); From 2e0370367cacd89a76f4a75954f70dc1f070ff07 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 26 Sep 2019 14:02:20 +0200 Subject: [PATCH 37/38] Print when we upgrade gravity database version. This will make possibly failed upgrades easier to debug. Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index a82d0d51..0fe90d8a 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -22,6 +22,7 @@ upgrade_gravityDB(){ if [[ "$version" == "1" ]]; then # This migration script upgrades the gravity.db file by # adding the domain_audit table + echo -e " ${INFO} Upgrading gravity database from version 1 to 2" sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" version=2 @@ -36,6 +37,7 @@ upgrade_gravityDB(){ # This migration script upgrades the gravity.db file by # renaming the regex table to regex_blacklist, and # creating a new regex_whitelist table + corresponding linking table and views + echo -e " ${INFO} Upgrading gravity database from version 2 to 3" sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/2_to_3.sql" version=3 fi From 149fb0c2160919f0cb1be0308c4bda7cf4fc6f23 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 27 Sep 2019 23:02:29 +0200 Subject: [PATCH 38/38] Do not install a blank regex file. Signed-off-by: DL6ER --- automated install/basic-install.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 19a6d919..cc78afbf 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -70,7 +70,6 @@ PI_HOLE_BLOCKPAGE_DIR="${webroot}/pihole" useUpdateVars=false adlistFile="/etc/pihole/adlists.list" -regexFile="/etc/pihole/regex.list" # Pi-hole needs an IP address; to begin, these variables are empty since we don't know what the IP is until # this script can run IPV4_ADDRESS="" @@ -1385,11 +1384,6 @@ installConfigs() { return 1 fi fi - # Install an empty regex file - if [[ ! -f "${regexFile}" ]]; then - # Let PHP edit the regex file, if installed - install -o pihole -g "${LIGHTTPD_GROUP:-pihole}" -m 664 /dev/null "${regexFile}" - fi # If the user chose to install the dashboard, if [[ "${INSTALL_WEB_SERVER}" == true ]]; then # and if the Web server conf directory does not exist,