From dfbf1a952feaeab5f2d6aeabef6ba3fcd79a5446 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 10 Oct 2015 13:51:21 -0500 Subject: [PATCH 1/9] deleting tabs and spaces --- gravity.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 1f5e3fb6..7e41dd94 100755 --- a/gravity.sh +++ b/gravity.sh @@ -1,6 +1,6 @@ #!/bin/bash # http://pi-hole.net -# Compiles a list of ad-serving domains by downloading them from multiple sources +# Compiles a list of ad-serving domains by downloading them from multiple sources # This script should only be run after you have a static IP address set on the Pi piholeIP=$(hostname -I) @@ -58,7 +58,7 @@ function createSwapFile() sudo dphys-swapfile setup sudo dphys-swapfile swapon } - + if [[ -n "$noSwap" ]]; then # if $noSwap is set, don't do anything @@ -83,20 +83,20 @@ do url=${sources[$i]} # Get just the domain from the URL domain=$(echo "$url" | cut -d'/' -f3) - + # Save the file as list.#.domain saveLocation=$origin/list.$i.$domain.$justDomainsExtension - + echo -n "Getting $domain list... " # Use a case statement to download lists that need special cURL commands to complete properly case "$domain" in "adblock.mahakala.is") data=$(curl -s -A 'Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' -e http://forum.xda-developers.com/ -z $saveLocation $url);; - + "pgl.yoyo.org") data=$(curl -s -d mimetype=plaintext -d hostformat=hosts -z $saveLocation $url);; *) data=$(curl -s -z $saveLocation -A "Mozilla/10.0" $url);; esac - + if [[ -n "$data" ]];then # Remove comments and print only the domain name # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious @@ -127,7 +127,7 @@ function gravity_advanced() ########################### { numberOf=$(cat $origin/$andLight | sed '/^\s*$/d' | wc -l) - echo "** $numberOf domains being pulled in by gravity..." + echo "** $numberOf domains being pulled in by gravity..." # Remove carriage returns and preceding whitespace cat $origin/$andLight | sed $'s/\r$//' | sed '/^\s*$/d' > $origin/$supernova # Sort and remove duplicates @@ -141,7 +141,7 @@ function gravity_advanced() sudo cp $origin/$accretionDisc $adList kill -HUP $(pidof dnsmasq) } - + # Whitelist (if applicable) then remove duplicates and format for dnsmasq if [[ -f $whitelist ]];then # Remove whitelist entries From e3c0fdfeb77d7f5fa8479a27aa87222141fc14c8 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 10 Oct 2015 13:52:54 -0500 Subject: [PATCH 2/9] new variables for IPv4 and IPv6 The IPv6 address looks for the one created from the MAC address. --- gravity.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 7e41dd94..55185954 100755 --- a/gravity.sh +++ b/gravity.sh @@ -3,7 +3,8 @@ # Compiles a list of ad-serving domains by downloading them from multiple sources # This script should only be run after you have a static IP address set on the Pi -piholeIP=$(hostname -I) +piholeIPv4=$(ip addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1") +piholeIPv6=$(ip addr show | awk '/scope\ global/ && /ff:fe/ {print $2}' | cut -d'/' -f1) # Ad-list sources--one per line in single quotes sources=('https://adaway.org/hosts.txt' From 88e6ce041d112be9ccae654bad3309c7f59a308e Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 10 Oct 2015 13:55:49 -0500 Subject: [PATCH 3/9] new command to print both the IPv4 and IPv6 address on their own line I'm not 100% sure if you can add multiple IP addresses to one domain on a single line, so I just broke it up into two. It makes the file bigger and longer, but I'll have to see how it works --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 55185954..06f41eef 100755 --- a/gravity.sh +++ b/gravity.sh @@ -137,7 +137,7 @@ function gravity_advanced() echo "** $numberOf unique domains trapped in the event horizon." # Format domain list as "192.168.x.x domain.com" echo "** Formatting domains into a HOSTS file..." - cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"'" $0}' > $origin/$accretionDisc + cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIPv4 "'" $0"\n""'"$piholeIPv6 "'" $0}' # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it sudo cp $origin/$accretionDisc $adList kill -HUP $(pidof dnsmasq) From 428d5e9a060d889b2d43e5caca27c56f65d53f9b Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 10 Oct 2015 14:01:10 -0500 Subject: [PATCH 4/9] Revert "new command to print both the IPv4 and IPv6 address on their own line" This reverts commit 88e6ce041d112be9ccae654bad3309c7f59a308e. --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 06f41eef..55185954 100755 --- a/gravity.sh +++ b/gravity.sh @@ -137,7 +137,7 @@ function gravity_advanced() echo "** $numberOf unique domains trapped in the event horizon." # Format domain list as "192.168.x.x domain.com" echo "** Formatting domains into a HOSTS file..." - cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIPv4 "'" $0"\n""'"$piholeIPv6 "'" $0}' + cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"'" $0}' > $origin/$accretionDisc # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it sudo cp $origin/$accretionDisc $adList kill -HUP $(pidof dnsmasq) From ee40bc04231fb34d7819a07ff20f7c324b94f319 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 10 Oct 2015 14:02:18 -0500 Subject: [PATCH 5/9] forgot to redirect it to the file --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 55185954..9ca3394b 100755 --- a/gravity.sh +++ b/gravity.sh @@ -137,7 +137,7 @@ function gravity_advanced() echo "** $numberOf unique domains trapped in the event horizon." # Format domain list as "192.168.x.x domain.com" echo "** Formatting domains into a HOSTS file..." - cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"'" $0}' > $origin/$accretionDisc + cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIPv4 "'" $0"\n""'"$piholeIPv6 "'" $0}' > $origin/$accretionDisc # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it sudo cp $origin/$accretionDisc $adList kill -HUP $(pidof dnsmasq) From 28dd956449bb2eef268846ef77d2d4db50e24a3f Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 17 Oct 2015 12:04:49 -0500 Subject: [PATCH 6/9] changing web server root folder Since the version of lighted in Jessie moved the default folder to /var/www/html, I am going to make it the same for wheezy for better compatibility. Also removed spaces --- advanced/lighttpd.conf | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/advanced/lighttpd.conf b/advanced/lighttpd.conf index 1c3ed076..a103c3ee 100644 --- a/advanced/lighttpd.conf +++ b/advanced/lighttpd.conf @@ -4,35 +4,35 @@ server.modules = ( "mod_redirect", "mod_rewrite" ) - -server.document-root = "/var/www" + +server.document-root = "/var/www/html" server.upload-dirs = ( "/var/cache/lighttpd/uploads" ) server.errorlog = "/var/log/lighttpd/error.log" server.pid-file = "/var/run/lighttpd.pid" server.username = "www-data" server.groupname = "www-data" server.port = 80 - - + + index-file.names = ( "index.php", "index.html", "index.lighttpd.html" ) url.access-deny = ( "~", ".inc" ) static-file.exclude-extensions = ( ".php", ".pl", ".fcgi" ) - + compress.cache-dir = "/var/cache/lighttpd/compress/" compress.filetype = ( "application/javascript", "text/css", "text/html", "text/plain" ) - + # default listening port for IPv6 falls back to the IPv4 port include_shell "/usr/share/lighttpd/use-ipv6.pl " + server.port include_shell "/usr/share/lighttpd/create-mime.assign.pl" include_shell "/usr/share/lighttpd/include-conf-enabled.pl" - + # Set access to 1 day for better query performance when the list gets so large # http://jacobsalmela.com/raspberry-pi-block-ads-adtrap/#comment-2013820434 $HTTP["url"] =~ "^/pihole/" { expire.url = ("" => "access plus 1 days") } - + # Rewrites all URLs to the /var/www/pihole/index.html $HTTP["host"] =~ ".*" { url.rewrite = (".*" => "pihole/index.html") -} \ No newline at end of file +} From 7b16a42f311e87532d9bc8cf13d4d6adccab419b Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 17 Oct 2015 12:11:03 -0500 Subject: [PATCH 7/9] conditionals for accessing the web interface This adds in some back end improvements for better debugging. It also allows you to access the Web interface, while you are connected to the Pi-hole, which resolves #31 --- advanced/lighttpd.conf | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/advanced/lighttpd.conf b/advanced/lighttpd.conf index a103c3ee..37763f97 100644 --- a/advanced/lighttpd.conf +++ b/advanced/lighttpd.conf @@ -2,6 +2,7 @@ server.modules = ( "mod_expire", "mod_compress", "mod_redirect", + "mod_setenv", "mod_rewrite" ) @@ -26,13 +27,20 @@ include_shell "/usr/share/lighttpd/use-ipv6.pl " + server.port include_shell "/usr/share/lighttpd/create-mime.assign.pl" include_shell "/usr/share/lighttpd/include-conf-enabled.pl" -# Set access to 1 day for better query performance when the list gets so large -# http://jacobsalmela.com/raspberry-pi-block-ads-adtrap/#comment-2013820434 -$HTTP["url"] =~ "^/pihole/" { - expire.url = ("" => "access plus 1 days") +# If the URL starts with /admin, it is the Web interface +$HTTP["url"] =~ "^/admin/" { + # Create a response header for debugging using curl -I + setenv.add-response-header = ( "X-Pi-hole" => "The Pi-hole Web interface is working!" ) } -# Rewrites all URLs to the /var/www/pihole/index.html -$HTTP["host"] =~ ".*" { - url.rewrite = (".*" => "pihole/index.html") +# If the URL does not start with /admin, then it is a query for an ad domain +$HTTP["url"] =~ "^(?!/admin)/.*" { + # Create a response header for debugging using curl -I + setenv.add-response-header = ( "X-Pi-hole" => "A black hole for Internet advertisements." ) + + # Set the cache to 1 day for better performance + expire.url = ("" => "access plus 1 days") + + # Send the query into the black hole + url.rewrite = (".*" => "pihole/index.html" ) } From 17904b31063931adc2d0a79a3a1b78424b6ec1c2 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Sat, 17 Oct 2015 12:12:51 -0500 Subject: [PATCH 8/9] ads still get blocked when a 404 happens --- advanced/lighttpd.conf | 1 + 1 file changed, 1 insertion(+) diff --git a/advanced/lighttpd.conf b/advanced/lighttpd.conf index 37763f97..a4253a0e 100644 --- a/advanced/lighttpd.conf +++ b/advanced/lighttpd.conf @@ -7,6 +7,7 @@ server.modules = ( ) server.document-root = "/var/www/html" +server.error-handler-404 = "pihole/index.html" server.upload-dirs = ( "/var/cache/lighttpd/uploads" ) server.errorlog = "/var/log/lighttpd/error.log" server.pid-file = "/var/run/lighttpd.pid" From 4ad2b68ee18923ddda438fad4e5d016b71eecce0 Mon Sep 17 00:00:00 2001 From: Jacob Salmela Date: Tue, 27 Oct 2015 17:16:17 -0500 Subject: [PATCH 9/9] reverting IPv6 so I can merge lighttpd.conf --- gravity.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 9ca3394b..f1b7aea0 100755 --- a/gravity.sh +++ b/gravity.sh @@ -3,13 +3,12 @@ # Compiles a list of ad-serving domains by downloading them from multiple sources # This script should only be run after you have a static IP address set on the Pi -piholeIPv4=$(ip addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1") -piholeIPv6=$(ip addr show | awk '/scope\ global/ && /ff:fe/ {print $2}' | cut -d'/' -f1) +piholeIP=$(hostname -I) # Ad-list sources--one per line in single quotes sources=('https://adaway.org/hosts.txt' 'http://adblock.gjtech.net/?format=unix-hosts' -'http://adblock.mahakala.is/' +#'http://adblock.mahakala.is/' 'http://hosts-file.net/.%5Cad_servers.txt' 'http://www.malwaredomainlist.com/hostslist/hosts.txt' 'http://pgl.yoyo.org/adservers/serverlist.php?' @@ -137,7 +136,7 @@ function gravity_advanced() echo "** $numberOf unique domains trapped in the event horizon." # Format domain list as "192.168.x.x domain.com" echo "** Formatting domains into a HOSTS file..." - cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIPv4 "'" $0"\n""'"$piholeIPv6 "'" $0}' > $origin/$accretionDisc + cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"'" $0}' > $origin/$accretionDisc # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it sudo cp $origin/$accretionDisc $adList kill -HUP $(pidof dnsmasq)