diff --git a/README.md b/README.md index 3e5e9253..be0d4369 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ A more detailed explanation of the installation can be found [here](http://jacob [![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif "AdminLTE Presentation")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=3J2L3Z4DHW9UY "Donate") ## Gravity -The [gravity.sh](https://github.com/jacobsalmela/pi-hole/blob/master/gravity.sh) does most of the magic. The script pulls in ad domains from many sources and compiles them into a single list of [over 900,000 entries](http://jacobsalmela.com/block-millions-ads-network-wide-with-a-raspberry-pi-hole-2-0). +The [gravity.sh](https://github.com/jacobsalmela/pi-hole/blob/master/gravity.sh) does most of the magic. The script pulls in ad domains from many sources and compiles them into a single list of [over 1.6 million entries](http://jacobsalmela.com/block-millions-ads-network-wide-with-a-raspberry-pi-hole-2-0). ## Whitelist and blacklist You can add a `whitelist.txt` or `blacklist.txt` in `/etc/pihole/` and the script will apply those files automatically. @@ -47,4 +47,4 @@ A technical and detailed description can be found [here](http://jacobsalmela.com ## Other Operating Systems This script will work for other UNIX-like systems with some slight **modifications**. As long as you can install `dnsmasq` and a Webserver, it should work OK. The automated install only works for a clean install of Raspiban right now since that is how the project originated. -[![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif "AdminLTE Presentation")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=3J2L3Z4DHW9UY "Donate") \ No newline at end of file +[![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif "AdminLTE Presentation")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=3J2L3Z4DHW9UY "Donate") diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index c745c502..80bb7543 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -27,12 +27,12 @@ read if [[ -f /etc/dnsmasq.d/adList.conf ]];then echo "Original Pi-hole detected. Initiating sub space transport..." sudo mkdir -p /etc/pihole/original/ - mv /etc/dnsmasq.d/adList.conf /etc/pihole/original/adList.conf.$(date "+%Y-%m-%d") - mv /etc/dnsmasq.conf /etc/pihole/original/dnsmasq.conf.$(date "+%Y-%m-%d") - mv /etc/resolv.conf /etc/pihole/original/resolv.conf.$(date "+%Y-%m-%d") - mv /etc/lighttpd/lighttpd.conf /etc/pihole/original/lighttpd.conf.$(date "+%Y-%m-%d") - mv /var/www/pihole/index.html /etc/pihole/original/index.html.$(date "+%Y-%m-%d") - mv /usr/local/bin/gravity.sh /etc/pihole/original/gravity.sh.$(date "+%Y-%m-%d") + sudo mv /etc/dnsmasq.d/adList.conf /etc/pihole/original/adList.conf.$(date "+%Y-%m-%d") + sudo mv /etc/dnsmasq.conf /etc/pihole/original/dnsmasq.conf.$(date "+%Y-%m-%d") + sudo mv /etc/resolv.conf /etc/pihole/original/resolv.conf.$(date "+%Y-%m-%d") + sudo mv /etc/lighttpd/lighttpd.conf /etc/pihole/original/lighttpd.conf.$(date "+%Y-%m-%d") + sudo mv /var/www/pihole/index.html /etc/pihole/original/index.html.$(date "+%Y-%m-%d") + sudo mv /usr/local/bin/gravity.sh /etc/pihole/original/gravity.sh.$(date "+%Y-%m-%d") else : fi @@ -41,6 +41,11 @@ echo "Updating the Pi..." sudo apt-get update sudo apt-get -y upgrade +echo "Installing tools..." +sudo apt-get -y install dnsutils +sudo apt-get -y install bc +sudo apt-get -y install toilet + echo "Installing DNS..." sudo apt-get -y install dnsmasq sudo update-rc.d dnsmasq enable diff --git a/gravity.sh b/gravity.sh index 71b194dd..1f5e3fb6 100755 --- a/gravity.sh +++ b/gravity.sh @@ -19,9 +19,6 @@ sources=('https://adaway.org/hosts.txt' adList=/etc/pihole/gravity.list origin=/etc/pihole piholeDir=/etc/pihole -if [[ -f $piholeDir/pihole.conf ]];then - . $piholeDir/pihole.conf -fi justDomainsExtension=domains matter=pihole.0.matter.txt andLight=pihole.1.andLight.txt @@ -34,6 +31,13 @@ latentBlacklist=$origin/latentBlacklist.txt whitelist=$piholeDir/whitelist.txt latentWhitelist=$origin/latentWhitelist.txt +# After setting defaults, check if there's local overrides +if [[ -r $piholeDir/pihole.conf ]];then + echo "** Local calibration requested..." + . $piholeDir/pihole.conf +fi + + echo "** Neutrino emissions detected..." # Create the pihole resource directory if it doesn't exist. Future files will be stored here @@ -55,7 +59,11 @@ function createSwapFile() sudo dphys-swapfile swapon } -if [[ -f /etc/dphys-swapfile ]];then + +if [[ -n "$noSwap" ]]; then + # if $noSwap is set, don't do anything + : +elif [[ -f /etc/dphys-swapfile ]];then swapSize=$(cat /etc/dphys-swapfile | grep -m1 CONF_SWAPSIZE | cut -d'=' -f2) if [[ $swapSize != 500 ]];then mv /etc/dphys-swapfile /etc/dphys-swapfile.orig @@ -72,30 +80,33 @@ fi # Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and blank lines for ((i = 0; i < "${#sources[@]}"; i++)) do + url=${sources[$i]} # Get just the domain from the URL - domain=$(echo "${sources[$i]}" | cut -d'/' -f3) + domain=$(echo "$url" | cut -d'/' -f3) # Save the file as list.#.domain - saveLocation=$origin/"list"."$i"."$domain" + saveLocation=$origin/list.$i.$domain.$justDomainsExtension + echo -n "Getting $domain list... " # Use a case statement to download lists that need special cURL commands to complete properly case "$domain" in - "adblock.mahakala.is") data=$(curl -s -A 'Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' -e http://forum.xda-developers.com/ -z $saveLocation."$justDomainsExtension" "${sources[$i]}");; + "adblock.mahakala.is") data=$(curl -s -A 'Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' -e http://forum.xda-developers.com/ -z $saveLocation $url);; - "pgl.yoyo.org") data=$(curl -s -d mimetype=plaintext -d hostformat=hosts -z $saveLocation."$justDomainsExtension" "${sources[$i]}");; + "pgl.yoyo.org") data=$(curl -s -d mimetype=plaintext -d hostformat=hosts -z $saveLocation $url);; - *) data=$(curl -s -z $saveLocation."$justDomainsExtension" -A "Mozilla/10.0" "${sources[$i]}");; + *) data=$(curl -s -z $saveLocation -A "Mozilla/10.0" $url);; esac if [[ -n "$data" ]];then - echo "Getting $domain list..." # Remove comments and print only the domain name # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious # This helps with that and makes it easier to read # It also helps with debugging so each stage of the script can be researched more in depth - echo "$data" | awk 'NF {if ($1 !~ "#") { if (NF>1) {print $2} else {print $1}}}' > $saveLocation."$justDomainsExtension" + echo "$data" | awk 'NF {if ($1 !~ "#") { if (NF>1) {print $2} else {print $1}}}' | \ + sed -e 's/^[. \t]*//' -e 's/\.\.\+/./g' -e 's/[. \t]*$//' | grep "\." > $saveLocation + echo "Done." else - echo "Skipping $domain list because it does not have any new entries..." + echo "Skipping list because it does not have any new entries." fi done @@ -105,11 +116,11 @@ find $origin/ -type f -name "*.$justDomainsExtension" -exec cat {} \; | tr -d '\ # Append blacklist entries if they exist if [[ -f $blacklist ]];then - numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l) - echo "** Blacklisting $numberOf domain(s)..." - cat $blacklist >> $origin/$matter + numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l) + echo "** Blacklisting $numberOf domain(s)..." + cat $blacklist >> $origin/$matter else - : + : fi function gravity_advanced() @@ -135,13 +146,24 @@ function gravity_advanced() if [[ -f $whitelist ]];then # Remove whitelist entries numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) - echo "** Whitelisting $numberOf domain(s)..." - # Append a "$" to the end of each line so it can be parsed out with grep -w - echo -n "^$" > $latentWhitelist - awk -F '[# \t]' 'NF>0&&$1!="" {print $1"$"}' $whitelist > $latentWhitelist - cat $origin/$matter | grep -vwf $latentWhitelist > $origin/$andLight - gravity_advanced + plural=; [[ "$numberOf" != "1" ]] && plural=s + echo "** Whitelisting $numberOf domain${plural}..." + # Append a "$" to the end, prepend a "^" to the beginning, and + # replace "." with "\." of each line to turn each entry into a + # regexp so it can be parsed out with grep -x + awk -F '[# \t]' 'NF>0&&$1!="" {print "^"$1"$"}' $whitelist | sed 's/\./\\./g' > $latentWhitelist else - cat $origin/$matter > $origin/$andLight - gravity_advanced + rm $latentWhitelist fi + +# Prevent our sources from being pulled into the hole +plural=; [[ "${#sources[@]}" != "1" ]] && plural=s +echo "** Whitelisting ${#sources[@]} ad list source${plural}..." +for url in ${sources[@]} +do + echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist +done + +grep -vxf $latentWhitelist $origin/$matter > $origin/$andLight + +gravity_advanced