From d5ffec64f25847197c082e37f18e78a194923523 Mon Sep 17 00:00:00 2001 From: Sven Giermann Date: Thu, 22 Oct 2015 15:17:44 +0200 Subject: [PATCH 01/15] add whitespace after $piholeIP In my case there had been no whitespace in piholeIP (I configured it statically because 'hostname -I' returns multiple IP addresses, which ends in errornous hosts file), so it's more safe to add one between IP and hostname. Further, I removed the obsolete latentBlacklist variable. --- gravity.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 12c8b0fd..d8c81809 100755 --- a/gravity.sh +++ b/gravity.sh @@ -27,7 +27,6 @@ eventHorizon=pihole.3.eventHorizon.txt accretionDisc=pihole.4.accretionDisc.txt eyeOfTheNeedle=pihole.5.wormhole.txt blacklist=$piholeDir/blacklist.txt -latentBlacklist=$origin/latentBlacklist.txt whitelist=$piholeDir/whitelist.txt latentWhitelist=$origin/latentWhitelist.txt @@ -136,7 +135,7 @@ function gravity_advanced() echo "** $numberOf unique domains trapped in the event horizon." # Format domain list as "192.168.x.x domain.com" echo "** Formatting domains into a HOSTS file..." - cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"'" $0}' > $origin/$accretionDisc + cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"' " $0}' > $origin/$accretionDisc # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it sudo cp $origin/$accretionDisc $adList kill -HUP $(pidof dnsmasq) From deb48c124fca132cbd81bd3b17eacd3f51c4ec44 Mon Sep 17 00:00:00 2001 From: Nathan Jones Date: Sun, 25 Oct 2015 16:53:20 -0700 Subject: [PATCH 02/15] Improve whitelisting script Match domain exactly instead of partially. Gather list of domains to minimize number of times that hosts file must be enumerated. Only add domain to whitelist if it isn't already present. --- advanced/Scripts/whitelist.sh | 47 +++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/whitelist.sh b/advanced/Scripts/whitelist.sh index aed6bd0f..d610d0a5 100755 --- a/advanced/Scripts/whitelist.sh +++ b/advanced/Scripts/whitelist.sh @@ -1,13 +1,44 @@ #!/bin/bash + +if [ $# = 0 ]; then + echo "Immediately whitelists one or more domains." + echo "Usage: whitelist.sh domain1 [domain2 ...]" +fi + +combopattern="" + # For each argument passed to this script for var in "$@" do - echo "Whitelisting $var..." - # Use sed to search for the domain in /etc/pihole/gravity.list and remove it using an in-place edit - sed -i "/$var/d" /etc/pihole/gravity.list - # Also add the domain to the whitelist.txt in /etc/pihole - echo "$var" >> /etc/pihole/whitelist.txt + echo "Whitelisting $var..." + + # Construct basic pattern to match domain name. + basicpattern=$(echo $var | awk -F '[# \t]' 'NF>0&&$1!="" {print ""$1""}' | sed 's/\./\\./g') + + if [ "$basicpattern" != "" ]; then + # Add to the combination pattern that will be used below + if [ "$combopattern" != "" ]; then combopattern="$combopattern|"; fi + combopattern="$combopattern$basicpattern" + + # Also add the domain to the whitelist but only if it's not already present + grep -E -q "^$basicpattern$" /etc/pihole/whitelist.txt \ + || echo "$var" >> /etc/pihole/whitelist.txt + fi done -echo "** $# domain(s) whitelisted." -# Force dnsmasq to reload /etc/pihole/gravity.list -kill -HUP $(pidof dnsmasq) \ No newline at end of file + +# Now report on and remove matched domains +if [ "$combopattern" != "" ]; then + echo "Modifying hosts file..." + + # Construct pattern to match entry in hosts file. + # This consists of one or more IP addresses followed by the domain name. + pattern=$(echo $combopattern | awk -F '[# \t]' '{printf "%s", "^(([0-9]+\.){3}[0-9]+ +)+("$1")$"}') + + # Output what will be removed and then actually remove + sed -r -n 's/'"$pattern"'/ Removed: \3/p' /etc/pihole/gravity.list + sed -r -i '/'"$pattern"'/d' /etc/pihole/gravity.list + + echo "** $# domain(s) whitelisted." + # Force dnsmasq to reload /etc/pihole/gravity.list + kill -HUP $(pidof dnsmasq) +fi From 3290dbbe486460dd582fc4f5d7695008fdbd9719 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 31 Oct 2015 17:09:21 -0500 Subject: [PATCH 03/15] link to Sky-Hole fork Pi-hole in the cloud with Sky-Hole. --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index ca6b5286..09b88524 100644 --- a/README.md +++ b/README.md @@ -60,4 +60,7 @@ A technical and detailed description can be found [here](http://jacobsalmela.com ## Other Operating Systems This script will work for other UNIX-like systems with some slight **modifications**. As long as you can install `dnsmasq` and a Webserver, it should work OK. The automated install only works for a clean install of Raspiban right now since that is how the project originated. +### Examples Of The Pi-hole On Other Operating Systems +- [Sky-Hole](http://dlaa.me/blog/post/skyhole) + [![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif "AdminLTE Presentation")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=3J2L3Z4DHW9UY "Donate") From 1b51b9ade358bcf6995a83d7a40e5ddb172d36a7 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Thu, 5 Nov 2015 14:31:38 -0600 Subject: [PATCH 04/15] for use as a cron job The log will be flushed daily and stats will be stored in a DB for displaying on the Web interface. --- advanced/Scripts/piholeLogFlush.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 advanced/Scripts/piholeLogFlush.sh diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh new file mode 100755 index 00000000..9647fe2c --- /dev/null +++ b/advanced/Scripts/piholeLogFlush.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# Flushes /var/log/pihole.log +truncate -s 0 /var/log/pihole.log From 23eece944308e75ca340b709d259cb15aabc28ad Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 5 Nov 2015 18:11:34 -0800 Subject: [PATCH 05/15] Upstream Branch pseudo-rebase --- gravity.sh | 100 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 61 insertions(+), 39 deletions(-) diff --git a/gravity.sh b/gravity.sh index f5bf6e8d..8e8da8e2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -1,6 +1,6 @@ #!/bin/bash # http://pi-hole.net -# Compiles a list of ad-serving domains by downloading them from multiple sources +# Compiles a list of ad-serving domains by downloading them from multiple sources # This script should only be run after you have a static IP address set on the Pi piholeIP=$(hostname -I) @@ -8,7 +8,7 @@ piholeIP=$(hostname -I) # Ad-list sources--one per line in single quotes sources=('https://adaway.org/hosts.txt' 'http://adblock.gjtech.net/?format=unix-hosts' -#'http://adblock.mahakala.is/' +'http://adblock.mahakala.is/' 'http://hosts-file.net/.%5Cad_servers.txt' 'http://www.malwaredomainlist.com/hostslist/hosts.txt' 'http://pgl.yoyo.org/adservers/serverlist.php?' @@ -19,6 +19,9 @@ sources=('https://adaway.org/hosts.txt' adList=/etc/pihole/gravity.list origin=/etc/pihole piholeDir=/etc/pihole +if [[ -f $piholeDir/pihole.conf ]];then + . $piholeDir/pihole.conf +fi justDomainsExtension=domains matter=pihole.0.matter.txt andLight=pihole.1.andLight.txt @@ -27,16 +30,10 @@ eventHorizon=pihole.3.eventHorizon.txt accretionDisc=pihole.4.accretionDisc.txt eyeOfTheNeedle=pihole.5.wormhole.txt blacklist=$piholeDir/blacklist.txt +latentBlacklist=$origin/latentBlacklist.txt whitelist=$piholeDir/whitelist.txt latentWhitelist=$origin/latentWhitelist.txt -# After setting defaults, check if there's local overrides -if [[ -r $piholeDir/pihole.conf ]];then - echo "** Local calibration requested..." - . $piholeDir/pihole.conf -fi - - echo "** Neutrino emissions detected..." # Create the pihole resource directory if it doesn't exist. Future files will be stored here @@ -57,12 +54,8 @@ function createSwapFile() sudo dphys-swapfile setup sudo dphys-swapfile swapon } - - -if [[ -n "$noSwap" ]]; then - # if $noSwap is set, don't do anything - : -elif [[ -f /etc/dphys-swapfile ]];then + +if [[ -f /etc/dphys-swapfile ]];then swapSize=$(cat /etc/dphys-swapfile | grep -m1 CONF_SWAPSIZE | cut -d'=' -f2) if [[ $swapSize != 500 ]];then mv /etc/dphys-swapfile /etc/dphys-swapfile.orig @@ -82,31 +75,55 @@ do url=${sources[$i]} # Get just the domain from the URL domain=$(echo "$url" | cut -d'/' -f3) - + # Save the file as list.#.domain saveLocation=$origin/list.$i.$domain.$justDomainsExtension - echo -n "Getting $domain list... " - # Use a case statement to download lists that need special cURL commands to complete properly + agent="Mozilla/10.0" + + echo -n "Getting $domain list... " + + # Use a case statement to download lists that need special cURL commands + # to complete properly and reset the user agent when required case "$domain" in - "adblock.mahakala.is") data=$(curl -s -A 'Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' -e http://forum.xda-developers.com/ -z $saveLocation $url);; + "adblock.mahakala.is") + agent='Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' + cmd="curl -e http://forum.xda-developers.com/" + ;; + + "pgl.yoyo.org") + cmd="curl -d mimetype=plaintext -d hostformat=hosts" + ;; - "pgl.yoyo.org") data=$(curl -s -d mimetype=plaintext -d hostformat=hosts -z $saveLocation $url);; - - *) data=$(curl -s -z $saveLocation -A "Mozilla/10.0" $url);; + # Default is a simple curl request + *) cmd="curl" esac - if [[ -n "$data" ]];then + # tmp file, so we don't have to store the (long!) lists in RAM + tmpfile=`mktemp` + timeCheck="" + if [ -r $saveLocation ]; then + timeCheck="-z $saveLocation" + fi + CMD="$cmd -s $timeCheck -A '$agent' $url > $tmpfile" + echo "running [$CMD]" + $cmd -s $timeCheck -A "$agent" $url > $tmpfile + + + if [[ -s "$tmpfile" ]];then # Remove comments and print only the domain name # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious # This helps with that and makes it easier to read # It also helps with debugging so each stage of the script can be researched more in depth - echo "$data" | awk 'NF {if ($1 !~ "#") { if (NF>1) {print $2} else {print $1}}}' | \ - sed -e 's/^[. \t]*//' -e 's/\.\.\+/./g' -e 's/[. \t]*$//' | grep "\." > $saveLocation + awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $tmpfile | \ + sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $saveLocation echo "Done." else echo "Skipping list because it does not have any new entries." fi + + # cleanup + rm -f $tmpfile done # Find all files with the .domains extension and compile them into one file and remove CRs @@ -114,39 +131,43 @@ echo "** Aggregating list of domains..." find $origin/ -type f -name "*.$justDomainsExtension" -exec cat {} \; | tr -d '\r' > $origin/$matter # Append blacklist entries if they exist -if [[ -f $blacklist ]];then +if [[ -r $blacklist ]];then numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l) echo "** Blacklisting $numberOf domain(s)..." cat $blacklist >> $origin/$matter -else - : fi -function gravity_advanced() ########################### - { - numberOf=$(cat $origin/$andLight | sed '/^\s*$/d' | wc -l) - echo "** $numberOf domains being pulled in by gravity..." +function gravity_advanced() { + + numberOf=$(wc -l $origin/$andLight) + echo "** $numberOf domains being pulled in by gravity..." + # Remove carriage returns and preceding whitespace - cat $origin/$andLight | sed $'s/\r$//' | sed '/^\s*$/d' > $origin/$supernova + # not really needed anymore? + cp $origin/$andLight $origin/$supernova + # Sort and remove duplicates - cat $origin/$supernova | sort | uniq > $origin/$eventHorizon - numberOf=$(cat $origin/$eventHorizon | sed '/^\s*$/d' | wc -l) + sort -u $origin/$supernova > $origin/$eventHorizon + numberOf=$(wc -l $origin/$eventHorizon) echo "** $numberOf unique domains trapped in the event horizon." + # Format domain list as "192.168.x.x domain.com" echo "** Formatting domains into a HOSTS file..." - cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"' " $0}' > $origin/$accretionDisc + awk '{print "'"$piholeIP"'" $1}' $origin/$eventHorizon > $origin/$accretionDisc + # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it sudo cp $origin/$accretionDisc $adList kill -HUP $(pidof dnsmasq) - } - +} + # Whitelist (if applicable) then remove duplicates and format for dnsmasq -if [[ -f $whitelist ]];then +if [[ -r $whitelist ]];then # Remove whitelist entries numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) plural=; [[ "$numberOf" != "1" ]] && plural=s echo "** Whitelisting $numberOf domain${plural}..." + # Append a "$" to the end, prepend a "^" to the beginning, and # replace "." with "\." of each line to turn each entry into a # regexp so it can be parsed out with grep -x @@ -163,6 +184,7 @@ do echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist done +# Remove whitelist entries from deduped list grep -vxf $latentWhitelist $origin/$matter > $origin/$andLight gravity_advanced From 8e21488542c2bbe32d6663243248b6d0f6ef56fe Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 5 Nov 2015 18:22:17 -0800 Subject: [PATCH 06/15] hawson memory reductions --- gravity.sh | 47 +++++++++++------------------------------------ 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/gravity.sh b/gravity.sh index 8e8da8e2..8ca34bb9 100755 --- a/gravity.sh +++ b/gravity.sh @@ -44,31 +44,6 @@ else sudo mkdir $piholeDir fi -# Add additional swap to prevent the "Error fork: unable to allocate memory" message: https://github.com/jacobsalmela/pi-hole/issues/37 -function createSwapFile() -######################### - { - echo "** Creating more swap space to accomodate large solar masses..." - sudo dphys-swapfile swapoff - sudo curl -s -o /etc/dphys-swapfile https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dphys-swapfile - sudo dphys-swapfile setup - sudo dphys-swapfile swapon - } - -if [[ -f /etc/dphys-swapfile ]];then - swapSize=$(cat /etc/dphys-swapfile | grep -m1 CONF_SWAPSIZE | cut -d'=' -f2) - if [[ $swapSize != 500 ]];then - mv /etc/dphys-swapfile /etc/dphys-swapfile.orig - echo "** Current swap size is $swapSize" - createSwapFile - else - : - fi -else - echo "** No swap file found. Creating one..." - createSwapFile -fi - # Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and blank lines for ((i = 0; i < "${#sources[@]}"; i++)) do @@ -100,22 +75,22 @@ do esac # tmp file, so we don't have to store the (long!) lists in RAM - tmpfile=`mktemp` - timeCheck="" + patternBuffer=`mktemp` + heisenbergCompensator="" if [ -r $saveLocation ]; then - timeCheck="-z $saveLocation" + heisenbergCompensator="-z $saveLocation" fi - CMD="$cmd -s $timeCheck -A '$agent' $url > $tmpfile" - echo "running [$CMD]" - $cmd -s $timeCheck -A "$agent" $url > $tmpfile + CMD="$cmd -s $heisenbergCompensator -A '$agent' $url > $patternBuffer" + echo "** Engaging pattern transference..." + $cmd -s $heisenbergCompensator -A "$agent" $url > $patternBuffer - if [[ -s "$tmpfile" ]];then + if [[ -s "$patternBuffer" ]];then # Remove comments and print only the domain name # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious # This helps with that and makes it easier to read # It also helps with debugging so each stage of the script can be researched more in depth - awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $tmpfile | \ + awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $patternBuffer | \ sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $saveLocation echo "Done." else @@ -123,7 +98,7 @@ do fi # cleanup - rm -f $tmpfile + rm -f $patternBuffer done # Find all files with the .domains extension and compile them into one file and remove CRs @@ -141,7 +116,7 @@ fi function gravity_advanced() { numberOf=$(wc -l $origin/$andLight) - echo "** $numberOf domains being pulled in by gravity..." + echo "** $numberOf $origin/$andLight domains being pulled in by gravity..." # Remove carriage returns and preceding whitespace # not really needed anymore? @@ -166,7 +141,7 @@ if [[ -r $whitelist ]];then # Remove whitelist entries numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) plural=; [[ "$numberOf" != "1" ]] && plural=s - echo "** Whitelisting $numberOf domain${plural}..." + echo "** Whitelisting $numberOf $whitelist domain${plural}..." # Append a "$" to the end, prepend a "^" to the beginning, and # replace "." with "\." of each line to turn each entry into a From 642c31c361acc6d3e433409322e018909ce51035 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Thu, 5 Nov 2015 18:33:05 -0800 Subject: [PATCH 07/15] Theme changes --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 8ca34bb9..e8c0acb7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -94,7 +94,7 @@ do sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $saveLocation echo "Done." else - echo "Skipping list because it does not have any new entries." + echo "Skipping pattern because transporter logic detected no changes..." fi # cleanup @@ -116,7 +116,7 @@ fi function gravity_advanced() { numberOf=$(wc -l $origin/$andLight) - echo "** $numberOf $origin/$andLight domains being pulled in by gravity..." + echo "** $numberOf domains being pulled in by gravity..." # Remove carriage returns and preceding whitespace # not really needed anymore? From 15b9fb68d7e86dcbc091a204ff862cabca02e312 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Fri, 6 Nov 2015 10:24:12 -0800 Subject: [PATCH 08/15] Remove filename from wc Line 118 and Line 127 changed wc to remove filename from output. --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index e8c0acb7..a4400a50 100755 --- a/gravity.sh +++ b/gravity.sh @@ -115,7 +115,7 @@ fi ########################### function gravity_advanced() { - numberOf=$(wc -l $origin/$andLight) + numberOf=$(wc -l < $origin/$andLight) echo "** $numberOf domains being pulled in by gravity..." # Remove carriage returns and preceding whitespace @@ -124,7 +124,7 @@ function gravity_advanced() { # Sort and remove duplicates sort -u $origin/$supernova > $origin/$eventHorizon - numberOf=$(wc -l $origin/$eventHorizon) + numberOf=$(wc -l < $origin/$eventHorizon) echo "** $numberOf unique domains trapped in the event horizon." # Format domain list as "192.168.x.x domain.com" From a06098bb1699b41b446892dba152b4751f807c34 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 6 Nov 2015 17:03:55 -0600 Subject: [PATCH 09/15] remove filename from whitelist output I also cleared the whitespace (or rather, Atom did). I moved the pinhole.conf to it's current location I commented out mahakala --- gravity.sh | 37 +++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/gravity.sh b/gravity.sh index a4400a50..33bf226e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -1,6 +1,6 @@ #!/bin/bash # http://pi-hole.net -# Compiles a list of ad-serving domains by downloading them from multiple sources +# Compiles a list of ad-serving domains by downloading them from multiple sources # This script should only be run after you have a static IP address set on the Pi piholeIP=$(hostname -I) @@ -8,7 +8,7 @@ piholeIP=$(hostname -I) # Ad-list sources--one per line in single quotes sources=('https://adaway.org/hosts.txt' 'http://adblock.gjtech.net/?format=unix-hosts' -'http://adblock.mahakala.is/' +#'http://adblock.mahakala.is/' 'http://hosts-file.net/.%5Cad_servers.txt' 'http://www.malwaredomainlist.com/hostslist/hosts.txt' 'http://pgl.yoyo.org/adservers/serverlist.php?' @@ -19,9 +19,6 @@ sources=('https://adaway.org/hosts.txt' adList=/etc/pihole/gravity.list origin=/etc/pihole piholeDir=/etc/pihole -if [[ -f $piholeDir/pihole.conf ]];then - . $piholeDir/pihole.conf -fi justDomainsExtension=domains matter=pihole.0.matter.txt andLight=pihole.1.andLight.txt @@ -30,10 +27,14 @@ eventHorizon=pihole.3.eventHorizon.txt accretionDisc=pihole.4.accretionDisc.txt eyeOfTheNeedle=pihole.5.wormhole.txt blacklist=$piholeDir/blacklist.txt -latentBlacklist=$origin/latentBlacklist.txt whitelist=$piholeDir/whitelist.txt latentWhitelist=$origin/latentWhitelist.txt +# After setting defaults, check if there's local overrides +if [[ -r $piholeDir/pihole.conf ]];then + echo "** Local calibration requested..." + . $piholeDir/pihole.conf +fi echo "** Neutrino emissions detected..." # Create the pihole resource directory if it doesn't exist. Future files will be stored here @@ -50,23 +51,23 @@ do url=${sources[$i]} # Get just the domain from the URL domain=$(echo "$url" | cut -d'/' -f3) - + # Save the file as list.#.domain saveLocation=$origin/list.$i.$domain.$justDomainsExtension agent="Mozilla/10.0" - + echo -n "Getting $domain list... " - # Use a case statement to download lists that need special cURL commands + # Use a case statement to download lists that need special cURL commands # to complete properly and reset the user agent when required case "$domain" in - "adblock.mahakala.is") + "adblock.mahakala.is") agent='Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' cmd="curl -e http://forum.xda-developers.com/" ;; - - "pgl.yoyo.org") + + "pgl.yoyo.org") cmd="curl -d mimetype=plaintext -d hostformat=hosts" ;; @@ -77,14 +78,14 @@ do # tmp file, so we don't have to store the (long!) lists in RAM patternBuffer=`mktemp` heisenbergCompensator="" - if [ -r $saveLocation ]; then + if [ -r $saveLocation ]; then heisenbergCompensator="-z $saveLocation" fi CMD="$cmd -s $heisenbergCompensator -A '$agent' $url > $patternBuffer" echo "** Engaging pattern transference..." $cmd -s $heisenbergCompensator -A "$agent" $url > $patternBuffer - + if [[ -s "$patternBuffer" ]];then # Remove comments and print only the domain name # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious @@ -116,11 +117,11 @@ fi function gravity_advanced() { numberOf=$(wc -l < $origin/$andLight) - echo "** $numberOf domains being pulled in by gravity..." + echo "** $numberOf domains being pulled in by gravity..." # Remove carriage returns and preceding whitespace # not really needed anymore? - cp $origin/$andLight $origin/$supernova + cp $origin/$andLight $origin/$supernova # Sort and remove duplicates sort -u $origin/$supernova > $origin/$eventHorizon @@ -135,13 +136,13 @@ function gravity_advanced() { sudo cp $origin/$accretionDisc $adList kill -HUP $(pidof dnsmasq) } - + # Whitelist (if applicable) then remove duplicates and format for dnsmasq if [[ -r $whitelist ]];then # Remove whitelist entries numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) plural=; [[ "$numberOf" != "1" ]] && plural=s - echo "** Whitelisting $numberOf $whitelist domain${plural}..." + echo "** Whitelisting $numberOf domain${plural}..." # Append a "$" to the end, prepend a "^" to the beginning, and # replace "." with "\." of each line to turn each entry into a From 1b65193fe40e9f1efe79765f0f762e3386d873d5 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 6 Nov 2015 17:05:04 -0600 Subject: [PATCH 10/15] new hostsfile url This seems to be the more appropriate URL: http://hosts-file.net/ad_servers.txt. I compared it to the original URL using opendiff and they are exactly the same. --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 33bf226e..2715ddd7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -9,7 +9,7 @@ piholeIP=$(hostname -I) sources=('https://adaway.org/hosts.txt' 'http://adblock.gjtech.net/?format=unix-hosts' #'http://adblock.mahakala.is/' -'http://hosts-file.net/.%5Cad_servers.txt' +'http://hosts-file.net/ad_servers.txt' 'http://www.malwaredomainlist.com/hostslist/hosts.txt' 'http://pgl.yoyo.org/adservers/serverlist.php?' 'http://someonewhocares.org/hosts/hosts' From c297a86211c69557f72bee3241422e3feceb5f67 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 6 Nov 2015 17:10:17 -0600 Subject: [PATCH 11/15] decluttering output I love the adherence to the theme, but this line makes it difficult to quickly read what is happening. --- gravity.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 2715ddd7..b157adb7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -82,7 +82,6 @@ do heisenbergCompensator="-z $saveLocation" fi CMD="$cmd -s $heisenbergCompensator -A '$agent' $url > $patternBuffer" - echo "** Engaging pattern transference..." $cmd -s $heisenbergCompensator -A "$agent" $url > $patternBuffer From 932d410b66e0fda80ea78b894c9427121d3d8b4a Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 6 Nov 2015 17:14:05 -0600 Subject: [PATCH 12/15] anal retentive changes OCD --- gravity.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index b157adb7..4c191c39 100755 --- a/gravity.sh +++ b/gravity.sh @@ -76,9 +76,9 @@ do esac # tmp file, so we don't have to store the (long!) lists in RAM - patternBuffer=`mktemp` + patternBuffer=$(mktemp) heisenbergCompensator="" - if [ -r $saveLocation ]; then + if [[ -r $saveLocation ]]; then heisenbergCompensator="-z $saveLocation" fi CMD="$cmd -s $heisenbergCompensator -A '$agent' $url > $patternBuffer" @@ -97,7 +97,7 @@ do echo "Skipping pattern because transporter logic detected no changes..." fi - # cleanup + # Cleanup rm -f $patternBuffer done From e1b8419a07a8a6feea408815c97986ecb44e9877 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 6 Nov 2015 17:16:14 -0600 Subject: [PATCH 13/15] theme addition Further adherence to the science/star trek theme --- gravity.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 4c191c39..b1024b09 100755 --- a/gravity.sh +++ b/gravity.sh @@ -75,7 +75,8 @@ do *) cmd="curl" esac - # tmp file, so we don't have to store the (long!) lists in RAM + echo "Narrowing the annular confinment beam..." + # Create a tmp file so we don't have to store the (long!) lists in RAM patternBuffer=$(mktemp) heisenbergCompensator="" if [[ -r $saveLocation ]]; then From a7d1b0b42c9dcc86b8ff274a4185d1ca78d47b50 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Fri, 6 Nov 2015 17:17:14 -0600 Subject: [PATCH 14/15] Revert "theme addition" This reverts commit e1b8419a07a8a6feea408815c97986ecb44e9877. --- gravity.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index b1024b09..4c191c39 100755 --- a/gravity.sh +++ b/gravity.sh @@ -75,8 +75,7 @@ do *) cmd="curl" esac - echo "Narrowing the annular confinment beam..." - # Create a tmp file so we don't have to store the (long!) lists in RAM + # tmp file, so we don't have to store the (long!) lists in RAM patternBuffer=$(mktemp) heisenbergCompensator="" if [[ -r $saveLocation ]]; then From ce0ca3cc61a138a37e1f285eb2f14a5e15ca603a Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 7 Nov 2015 07:58:35 -0600 Subject: [PATCH 15/15] for use with /etc/crontab Adding a system-wide crontab to update the ad source lists, flush the log daily, and (eventually) record the stats for historical purposes. --- advanced/pihole.cron | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/advanced/pihole.cron b/advanced/pihole.cron index a707607e..47bc61fb 100644 --- a/advanced/pihole.cron +++ b/advanced/pihole.cron @@ -1 +1,26 @@ -@weekly sudo /usr/local/bin/gravity.sh +# /etc/crontab: system-wide crontab +# Unlike any other crontab you don't have to run the `crontab' +# command to install the new version when you edit this file +# and files in /etc/cron.d. These files also have username fields, +# that none of the other crontabs do. + +SHELL=/bin/sh +PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin + +# m h dom mon dow user command +17 * * * * root cd / && run-parts --report /etc/cron.hourly +25 6 * * * root test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.daily ) +47 6 * * 7 root test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.weekly ) +52 6 1 * * root test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.monthly ) + +# Pi-hole: Update the ad sources once a week on Sunday at 01:59 +# Download any updates from the ad lists +59 1 * * 7 root /usr/local/bin/gravity.sh + +# Pi-hole: Parse the log file before it is flushed and save the stats to a database +# This will be used for a historical view of your Pi-hole's performance +#50 11 * * * root /usr/local/bin/dailyLog.sh + +# Pi-hole: Flush the log daily at 11:58 so it doesn't get out of control +# Stats will be viewable in the Web interface thanks to the cron job above +58 11 * * * root /usr/local/bin/piholeLogFlush.sh