From 86fa4e9ee83a283ca6b70be2c7c80ea78aa71c70 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 31 Mar 2017 18:55:07 +0100 Subject: [PATCH 01/13] trim down default adlist --- adlists.default | 55 +++++++++++-------------------------------------- 1 file changed, 12 insertions(+), 43 deletions(-) diff --git a/adlists.default b/adlists.default index 4a5bca3c..94c75beb 100644 --- a/adlists.default +++ b/adlists.default @@ -1,53 +1,22 @@ -## Pi-hole ad-list default sources. Updated 29/10/2016 ######################### -# # -# To make changes to this file: # -# 1. run `cp /etc/pihole/adlists.default /etc/pihole/adlists.list` # -# 2. run `nano /etc/pihole/adlists.list` # -# 3. Uncomment or comment any of the below lists # -# # -# Know of any other lists? Feel free to let us know about them, or add them # -# to this file! # -################################################################################ - # The below list amalgamates several lists we used previously. # See `https://github.com/StevenBlack/hosts` for details +##StevenBlack's list https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts -# Other lists we consider safe: +##MalwareDomains https://mirror1.malwaredomains.com/files/justdomains + +##Cameleon http://sysctl.org/cameleon/hosts + +##Zeustracker https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist + +##Disconnect.me Tracking https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt + +##Disconnect.me Ads https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt -# hosts-file.net list. Updated frequently, but has been known to block legitimate sites. -https://hosts-file.net/ad_servers.txt - -# Mahakala list. Has been known to block legitimate domains including the entire .com range. -# Warning: Due to the sheer size of this list, the web admin console will be unresponsive. -#https://adblock.mahakala.is/ - -# ADZHOSTS list. Has been known to block legitimate domains -#http://pilotfiber.dl.sourceforge.net/project/adzhosts/HOSTS.txt - -# Windows 10 telemetry list -#https://raw.githubusercontent.com/crazy-max/WindowsSpyBlocker/master/data/hosts/win10/spy.txt - -# Securemecca.com list - Also blocks "adult" sites (pornography/gambling etc) -#http://securemecca.com/Downloads/hosts.txt - -# Quidsup's tracker list -#https://raw.githubusercontent.com/quidsup/notrack/master/trackers.txt - -# Block the BBC News website Breaking News banner -#https://raw.githubusercontent.com/BreakingTheNews/BreakingTheNews.github.io/master/hosts - -# Untested Lists: -#https://raw.githubusercontent.com/reek/anti-adblock-killer/master/anti-adblock-killer-filters.txt -#https://raw.githubusercontent.com/Dawsey21/Lists/master/main-blacklist.txt -#http://malwaredomains.lehigh.edu/files/domains.txt -# Following two lists should be used simultaneously: (readme https://github.com/notracking/hosts-blocklists/) -#https://raw.github.com/notracking/hosts-blocklists/master/hostnames.txt -#https://raw.github.com/notracking/hosts-blocklists/master/domains.txt -# Combination of several host files on the internet (warning some facebook domains are also blocked but you can go to facebook.com). See https://github.com/mat1th/Dns-add-block for more information. -#https://raw.githubusercontent.com/mat1th/Dns-add-block/master/hosts +##Hosts-file.net +https://hosts-file.net/ad_servers.txt \ No newline at end of file From ce97896ffd9f9cfd7650804c84524978705403eb Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 31 Mar 2017 19:00:04 +0100 Subject: [PATCH 02/13] account for new `adlists.custom` file --- gravity.sh | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 7441dd87..af6c8599 100755 --- a/gravity.sh +++ b/gravity.sh @@ -30,6 +30,7 @@ PIHOLE_COMMAND="/usr/local/bin/pihole" adListFile=/etc/pihole/adlists.list adListDefault=/etc/pihole/adlists.default +adListCustom=/etc/pihole/adlists.custom whitelistScript="${PIHOLE_COMMAND} -w" whitelistFile=/etc/pihole/whitelist.txt blacklistFile=/etc/pihole/blacklist.txt @@ -75,8 +76,8 @@ gravity_collapse() { echo ":::" #Decide if we're using a custom ad block list, or defaults. if [ -f ${adListFile} ]; then - #custom file found, use this instead of default - echo -n "::: Custom adList file detected. Reading..." + #User has disabled one or more default lists + echo -n "::: Changes to default list detected. Reading adlists.list..." sources=() while IFS= read -r line || [[ -n "$line" ]]; do #Do not read commented out or blank lines @@ -88,8 +89,8 @@ gravity_collapse() { done < ${adListFile} echo " done!" else - #no custom file found, use defaults! - echo -n "::: No custom adlist file detected, reading from default file..." + # + echo -n "::: No changes to default list detected. Reading adlists.default..." sources=() while IFS= read -r line || [[ -n "$line" ]]; do #Do not read commented out or blank lines @@ -101,6 +102,19 @@ gravity_collapse() { done < ${adListDefault} echo " done!" fi + + if [ -f ${adListCustom} ]; then + echo -n "Custom additional lists detected. Reading adlists.custom..." + while IFS= read -r line || [[ -n "$line" ]]; do + #Do not read commented out or blank lines + if [[ ${line} = \#* ]] || [[ ! ${line} ]]; then + echo "" > /dev/null + else + sources+=(${line}) + fi + done < ${adListCustom} + echo " done!" + fi } # patternCheck - check to see if curl downloaded any new files. From e720de401d1a8dbe17006c94addb150ec1f782d6 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 31 Mar 2017 19:16:09 +0100 Subject: [PATCH 03/13] One file to rule them all, same functionality (made changes and want to go back to default? Delete your adlists.list) --- gravity.sh | 72 ++++++++++++++++++++++-------------------------------- 1 file changed, 29 insertions(+), 43 deletions(-) diff --git a/gravity.sh b/gravity.sh index af6c8599..17166a85 100755 --- a/gravity.sh +++ b/gravity.sh @@ -29,8 +29,8 @@ EOM PIHOLE_COMMAND="/usr/local/bin/pihole" adListFile=/etc/pihole/adlists.list -adListDefault=/etc/pihole/adlists.default -adListCustom=/etc/pihole/adlists.custom +adListDefault=/etc/pihole/adlists.default #being deprecated +adListRepoDefault=/etc/.pihole/adlists.default whitelistScript="${PIHOLE_COMMAND} -w" whitelistFile=/etc/pihole/whitelist.txt blacklistFile=/etc/pihole/blacklist.txt @@ -72,49 +72,35 @@ fi ########################### # collapse - begin formation of pihole gravity_collapse() { + + #New Logic: + # Does /etc/pihole/adlists.list exist? If so leave it alone + # If not, cp /etc/.pihole/adlists.default /etc/pihole/adlists.list + # Read from adlists.list + + #The following two blocks will sort out any missing adlists in the /etc/pihole directory, and remove legacy adlists.default + if [ -f ${adListDefault} ] && [ -f ${adListFile} ]; then + rm ${adListDefault} + fi + + if [ ! -f ${adListFile} ]; then + cp ${adListRepoDefault} ${adListFile} + fi + echo "::: Neutrino emissions detected..." echo ":::" - #Decide if we're using a custom ad block list, or defaults. - if [ -f ${adListFile} ]; then - #User has disabled one or more default lists - echo -n "::: Changes to default list detected. Reading adlists.list..." - sources=() - while IFS= read -r line || [[ -n "$line" ]]; do - #Do not read commented out or blank lines - if [[ ${line} = \#* ]] || [[ ! ${line} ]]; then - echo "" > /dev/null - else - sources+=(${line}) - fi - done < ${adListFile} - echo " done!" - else - # - echo -n "::: No changes to default list detected. Reading adlists.default..." - sources=() - while IFS= read -r line || [[ -n "$line" ]]; do - #Do not read commented out or blank lines - if [[ ${line} = \#* ]] || [[ ! ${line} ]]; then - echo "" > /dev/null - else - sources+=(${line}) - fi - done < ${adListDefault} - echo " done!" - fi - - if [ -f ${adListCustom} ]; then - echo -n "Custom additional lists detected. Reading adlists.custom..." - while IFS= read -r line || [[ -n "$line" ]]; do - #Do not read commented out or blank lines - if [[ ${line} = \#* ]] || [[ ! ${line} ]]; then - echo "" > /dev/null - else - sources+=(${line}) - fi - done < ${adListCustom} - echo " done!" - fi + #User has disabled one or more default lists + echo -n "::: Pulling source lists into range..." + sources=() + while IFS= read -r line || [[ -n "$line" ]]; do + #Do not read commented out or blank lines + if [[ ${line} = \#* ]] || [[ ! ${line} ]]; then + echo "" > /dev/null + else + sources+=(${line}) + fi + done < ${adListFile} + echo " done!" } # patternCheck - check to see if curl downloaded any new files. From bdc6bd41358680937c7d904fa51f5ae86a4239ad Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 31 Mar 2017 19:18:28 +0100 Subject: [PATCH 04/13] Don't need this comment anymore --- gravity.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 17166a85..05593d1e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -89,7 +89,6 @@ gravity_collapse() { echo "::: Neutrino emissions detected..." echo ":::" - #User has disabled one or more default lists echo -n "::: Pulling source lists into range..." sources=() while IFS= read -r line || [[ -n "$line" ]]; do From 02040cd25d4d4035fa0ae4b18ca63ad51ed34a9b Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 31 Mar 2017 19:23:26 +0100 Subject: [PATCH 05/13] We no longer need to do this, either --- gravity.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 05593d1e..34e03ca2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -403,8 +403,6 @@ if [[ "${forceGrav}" == true ]]; then echo " done!" fi -#Overwrite adlists.default from /etc/.pihole in case any changes have been made. Changes should be saved in /etc/adlists.list -cp /etc/.pihole/adlists.default /etc/pihole/adlists.default gravity_collapse gravity_spinup if [[ "${skipDownload}" == false ]]; then From b4bc90fb8519fec1e1c5b9585aa14996e5b84c13 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 31 Mar 2017 19:35:52 +0100 Subject: [PATCH 06/13] add in CustomizeAdlists function --- advanced/Scripts/webpage.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 9637cc6f..b15daf88 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -319,6 +319,22 @@ SetWebUILayout(){ } +CustomizeAdLists() { + + list = "/etc/pihole/adlists.list" + + if [[ "${args[2]}" == "enable" ]] ; then + sed -i "\\@${args[3]}@s/^#http/http/g" "${list}" + elif [[ "${args[2]}" == "disable" ]] ; then + sed -i "\\@${args[3]}@s/^http/#http/g" "${list}" + elif [[ "${args[2]}" == "add" && "${args[3]}" == "user" ]] ; then + echo "${args[3]}" >> /etc/pihole/adlists.user + else + echo "Not permitted" + return 1 + fi +} + SetPrivacyMode(){ if [[ "${args[2]}" == "true" ]] ; then From 3a3dde629896ba8a02c110e195bec08b19482bf0 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 31 Mar 2017 20:03:56 +0100 Subject: [PATCH 07/13] add adlist argument redirector --- advanced/Scripts/webpage.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index b15daf88..02568134 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -458,6 +458,7 @@ main() { "hostrecord" ) SetHostRecord;; "-i" | "interface" ) SetListeningMode;; "-t" | "teleporter" ) Teleporter;; + "adlist" ) CustomizeAdLists;; * ) helpFunc;; esac From 5435b93df2764f7d6e67749e8eac29d180abba27 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 31 Mar 2017 20:05:45 +0100 Subject: [PATCH 08/13] remove spaces --- advanced/Scripts/webpage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 02568134..887b8f34 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -321,7 +321,7 @@ SetWebUILayout(){ CustomizeAdLists() { - list = "/etc/pihole/adlists.list" + list="/etc/pihole/adlists.list" if [[ "${args[2]}" == "enable" ]] ; then sed -i "\\@${args[3]}@s/^#http/http/g" "${list}" From af2cff5177be159ad224b0dd2cb3403c375b3b02 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 31 Mar 2017 20:13:04 +0100 Subject: [PATCH 09/13] remove call to user list --- advanced/Scripts/webpage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 887b8f34..974dbdb5 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -327,8 +327,8 @@ CustomizeAdLists() { sed -i "\\@${args[3]}@s/^#http/http/g" "${list}" elif [[ "${args[2]}" == "disable" ]] ; then sed -i "\\@${args[3]}@s/^http/#http/g" "${list}" - elif [[ "${args[2]}" == "add" && "${args[3]}" == "user" ]] ; then - echo "${args[3]}" >> /etc/pihole/adlists.user + elif [[ "${args[2]}" == "add" ]] ; then + echo "${args[3]}" >> ${list} else echo "Not permitted" return 1 From 42393123a0341e8e00eadbdd14ff9dfeb7250489 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 31 Mar 2017 20:15:13 +0100 Subject: [PATCH 10/13] add newline to adlists.default --- adlists.default | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adlists.default b/adlists.default index 94c75beb..f91d7020 100644 --- a/adlists.default +++ b/adlists.default @@ -19,4 +19,4 @@ https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt ##Hosts-file.net -https://hosts-file.net/ad_servers.txt \ No newline at end of file +https://hosts-file.net/ad_servers.txt From 1f74b1e2fdff929ec280573210aa914d7554b03f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 31 Mar 2017 21:12:17 +0100 Subject: [PATCH 11/13] newline --- adlists.default | 1 + 1 file changed, 1 insertion(+) diff --git a/adlists.default b/adlists.default index f91d7020..cbc1bfb3 100644 --- a/adlists.default +++ b/adlists.default @@ -20,3 +20,4 @@ https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt ##Hosts-file.net https://hosts-file.net/ad_servers.txt + From 153031482f77c8ee4757093c8c5b94b3f8afc6f6 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 31 Mar 2017 22:51:10 +0100 Subject: [PATCH 12/13] Add delete mode to adlist command. Might be abetter way to do this --- advanced/Scripts/webpage.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 974dbdb5..f604a00f 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -329,6 +329,9 @@ CustomizeAdLists() { sed -i "\\@${args[3]}@s/^http/#http/g" "${list}" elif [[ "${args[2]}" == "add" ]] ; then echo "${args[3]}" >> ${list} + elif [[ "${args[2]}" == "del" ]] ; then + var=$(echo "${args[3]}" | sed 's/\//\\\//g') + sed -i "/${var}/Id" "${list}" else echo "Not permitted" return 1 From 3cf9942465d51a94035bedef72349e9889e22da2 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 31 Mar 2017 23:00:56 +0100 Subject: [PATCH 13/13] spacing fixed --- advanced/Scripts/webpage.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index f604a00f..5b121b44 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -329,9 +329,9 @@ CustomizeAdLists() { sed -i "\\@${args[3]}@s/^http/#http/g" "${list}" elif [[ "${args[2]}" == "add" ]] ; then echo "${args[3]}" >> ${list} - elif [[ "${args[2]}" == "del" ]] ; then - var=$(echo "${args[3]}" | sed 's/\//\\\//g') - sed -i "/${var}/Id" "${list}" + elif [[ "${args[2]}" == "del" ]] ; then + var=$(echo "${args[3]}" | sed 's/\//\\\//g') + sed -i "/${var}/Id" "${list}" else echo "Not permitted" return 1