diff --git a/README.md b/README.md index 3e02f0d4..258168c2 100644 --- a/README.md +++ b/README.md @@ -63,5 +63,6 @@ This script will work for other UNIX-like systems with some slight **modificatio ### Examples Of The Pi-hole On Other Operating Systems - [Sky-Hole](http://dlaa.me/blog/post/skyhole) +- [Pi-hole in the Cloud!](http://blog.codybunch.com/2015/07/28/Pi-Hole-in-the-cloud/) [![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif "AdminLTE Presentation")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=3J2L3Z4DHW9UY "Donate") diff --git a/advanced/Scripts/whitelist.sh b/advanced/Scripts/whitelist.sh index d610d0a5..cda557c0 100755 --- a/advanced/Scripts/whitelist.sh +++ b/advanced/Scripts/whitelist.sh @@ -1,6 +1,11 @@ #!/bin/bash +whitelist=/etc/pihole/whitelist.txt +adList=/etc/pihole/gravity.list +if [[ ! -f $whitelist ]];then + touch $whitelist +fi -if [ $# = 0 ]; then +if [[ $# = 0 ]]; then echo "Immediately whitelists one or more domains." echo "Usage: whitelist.sh domain1 [domain2 ...]" fi @@ -15,28 +20,28 @@ do # Construct basic pattern to match domain name. basicpattern=$(echo $var | awk -F '[# \t]' 'NF>0&&$1!="" {print ""$1""}' | sed 's/\./\\./g') - if [ "$basicpattern" != "" ]; then + if [[ "$basicpattern" != "" ]]; then # Add to the combination pattern that will be used below - if [ "$combopattern" != "" ]; then combopattern="$combopattern|"; fi + if [[ "$combopattern" != "" ]]; then combopattern="$combopattern|"; fi combopattern="$combopattern$basicpattern" # Also add the domain to the whitelist but only if it's not already present - grep -E -q "^$basicpattern$" /etc/pihole/whitelist.txt \ - || echo "$var" >> /etc/pihole/whitelist.txt + grep -E -q "^$basicpattern$" $whitelist \ + || echo "$var" >> $whitelist fi done # Now report on and remove matched domains -if [ "$combopattern" != "" ]; then +if [[ "$combopattern" != "" ]]; then echo "Modifying hosts file..." - + # Construct pattern to match entry in hosts file. # This consists of one or more IP addresses followed by the domain name. pattern=$(echo $combopattern | awk -F '[# \t]' '{printf "%s", "^(([0-9]+\.){3}[0-9]+ +)+("$1")$"}') # Output what will be removed and then actually remove - sed -r -n 's/'"$pattern"'/ Removed: \3/p' /etc/pihole/gravity.list - sed -r -i '/'"$pattern"'/d' /etc/pihole/gravity.list + sed -r -n 's/'"$pattern"'/ Removed: \3/p' $adList + sed -r -i '/'"$pattern"'/d' $adList echo "** $# domain(s) whitelisted." # Force dnsmasq to reload /etc/pihole/gravity.list diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index d5fef596..51285a4b 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/usr/bin/env bash # Pi-hole: A black hole for Internet advertisements # by Jacob Salmela # Network-wide ad blocking via your Raspberry Pi @@ -24,9 +24,11 @@ columns=$(stty -a | tr \; \\012 | egrep 'columns' | cut -d' ' -f3) r=$(( rows / 2 )) c=$(( columns / 2 )) -IPv4addr=$(ip -4 addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1") -IPv4mask=$(ifconfig | awk -F':' '/inet addr/ && !/127.0.0.1/ {print $4}') -IPv4gw=$(ip route show | awk '/default\ via/ {print $3}') +# Find IP used to route to outside world +IPv4info=$(ip route get 8.8.8.8) +IPv4dev=$(echo $IPv4info| awk '{print $5}') +IPv4addr=$(ip -o -f inet addr show dev $IPv4dev | awk '{print $4}') +IPv4gw=$(echo $IPv4info | awk '{print $3}') # IPv6 support to be added later #IPv6eui64=$(ip addr show | awk '/scope\ global/ && /ff:fe/ {print $2}' | cut -d'/' -f1) @@ -195,7 +197,6 @@ getStaticIPv4Settings() if (whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Do you want to use your current network settings as a static address? IP address: $IPv4addr - Subnet mask: $IPv4mask Gateway: $IPv4gw" $r $c) then # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict. whiptail --msgbox --backtitle "IP information" --title "FYI: IP Conflict" "It is possible your router could still try to assign this IP to a device, which would cause a conflict. But in most cases the router is smart enough to not do that. @@ -205,8 +206,6 @@ if (whiptail --backtitle "Calibrating network interface" --title "Static IP Addr It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." $r $c # Nothing else to do since the variables are already set above else - # Since a custom address will be used, restart at the end of the script to apply the new changes - rebootNeeded=true # Otherwise, we need to ask the user to input their desired settings. # Start by getting the IPv4 address (pre-filling it with info gathered from DHCP) # Start a loop to let the user enter their information with the chance to go back and edit it if necessary @@ -216,22 +215,17 @@ else IPv4addr=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" $r $c $IPv4addr 3>&1 1>&2 2>&3) if [[ $? = 0 ]];then echo "Your static IPv4 address: $IPv4addr" - # Ask for the subnet mask - IPv4mask=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 netmask" --inputbox "Enter your desired IPv4 subnet mask" $r $c $IPv4mask 3>&1 1>&2 2>&3) - if [[ $? = 0 ]];then - echo "Your static IPv4 netmask: $IPv4mask" - # Ask for the gateway + # Ask for the gateway IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" $r $c $IPv4gw 3>&1 1>&2 2>&3) if [[ $? = 0 ]];then echo "Your static IPv4 gateway: $IPv4gw" # Give the user a chance to review their settings before moving on if (whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Are these settings correct? IP address: $IPv4addr - Subnet mask: $IPv4mask Gateway: $IPv4gw" $r $c)then # If the settings are correct, then we need to set the piholeIP # Saving it to a temporary file us to retrieve it later when we run the gravity.sh script - echo $IPv4addr > /tmp/piholeIP + echo ${IPv4addr%/*} > /tmp/piholeIP # After that's done, the loop ends and we move on ipSettingsCorrect=True else @@ -245,12 +239,6 @@ else exit fi else - # Cancelling subnet mask settings window - ipSettingsCorrect=False - echo "User canceled." - exit - fi - else # Cancelling IPv4 settings window ipSettingsCorrect=False echo "User canceled." @@ -266,9 +254,10 @@ setStaticIPv4() { # Append these lines to /etc/dhcpcd.conf to enable a static IP echo "interface $piholeInterface -static ip_address=$IPv4addr/24 +static ip_address=$IPv4addr static routers=$IPv4gw static domain_name_servers=$IPv4gw" | sudo tee -a $dhcpcdFile >/dev/null +sudo ip addr replace dev $piholeInterface $IPv4addr } installPihole() @@ -358,10 +347,11 @@ sudo mv $tmpLog $instalLogLoc whiptail --msgbox --backtitle "Make it so." --title "Installation Complete!" "Configure your devices to use the Pi-hole as their DNS server using this IP: $IPv4addr. -If you didn't use DHCP settings as your new static address, the Pi will restart after this dialog. If you are using SSH, you may need to reconnect using the IP address above. +If you set a new IP address, it should work fine, but you may want to reboot the Pi at some point. The install log is in /etc/pihole." $r $c +<<<<<<< HEAD # If a custom address was set, restart if [[ "$rebootNeeded" = true ]];then # Restart to apply the new static IP address @@ -372,3 +362,7 @@ else sudo service lighttpd start fi >>>>>>> refs/remotes/jacobsalmela/master +======= +sudo service dnsmasq start +sudo service lighttpd start +>>>>>>> refs/remotes/jacobsalmela/master diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh new file mode 100644 index 00000000..b0dd3368 --- /dev/null +++ b/automated install/uninstall.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Completely uninstalls the Pi-hole + +######### SCRIPT ########### +sudo apt-get -y remove --purge dnsutils bc toilet +sudo apt-get -y remove --purge dnsmasq +sudo apt-get -y remove --purge lighttpd php5-common php5-cgi php5 +sudo rm -rf /var/www/html +sudo rm /etc/dnsmasq.conf /etc/dnsmasq.conf.orig +sudo rm /etc/crontab +sudo mv /etc/crontab.orig /etc/crontab +sudo rm /etc/dnsmasq.conf +sudo rm -rf /etc/lighttpd/ +sudo rm /var/log/pihole.log +sudo rm /usr/local/bin/gravity.sh +sudo rm /usr/local/bin/chronometer.sh +sudo rm /usr/local/bin/whitelist.sh +sudo rm /usr/local/bin/piholeLogFlush.sh +sudo rm -rf /etc/pihole/ diff --git a/gravity.sh b/gravity.sh index 3a0230c1..7eeecf76 100755 --- a/gravity.sh +++ b/gravity.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash +# Pi-hole: A black hole for Internet advertisements +# (c) 2015 by Jacob Salmela GPL 2.0 +# Network-wide ad blocking via your Raspberry Pi # http://pi-hole.net # Compiles a list of ad-serving domains by downloading them from multiple sources + piholeIPfile=/tmp/piholeIP if [[ -f $piholeIPfile ]];then # If the file exists, it means it was exported from the installation script and we should use that value instead of detecting it in this script @@ -8,7 +12,9 @@ if [[ -f $piholeIPfile ]];then rm $piholeIPfile else # Otherwise, the IP address can be taken directly from the machine, which will happen when the script is run by the user and not the installation script - piholeIP=$(ip -4 addr show | awk '{match($0,/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/); ip = substr($0,RSTART,RLENGTH); print ip}' | sed '/^\s*$/d' | grep -v "127.0.0.1") + IPv4dev=$(ip route get 8.8.8.8 | awk '{print $5}') + piholeIPCIDR=$(ip -o -f inet addr show dev $IPv4dev | awk '{print $4}') + piholeIP=${piholeIPCIDR%/*} fi #Checks if the script is being run as root and sets sudo accordingly @@ -22,7 +28,8 @@ exit 1 fi # Ad-list sources--one per line in single quotes -# The mahakala source is commented out due to many users having issues with it blocking legitimate domains. Uncomment at your own risk +# The mahakala source is commented out due to many users having issues with it blocking legitimate domains. +# Uncomment at your own risk sources=('https://adaway.org/hosts.txt' 'http://adblock.gjtech.net/?format=unix-hosts' #'http://adblock.mahakala.is/' @@ -33,23 +40,24 @@ sources=('https://adaway.org/hosts.txt' 'http://winhelp2002.mvps.org/hosts.txt') # Variables for various stages of downloading and formatting the list -adList=/etc/pihole/gravity.list -origin=/etc/pihole -piholeDir=/etc/pihole -justDomainsExtension=domains -matter=pihole.0.matter.txt -andLight=pihole.1.andLight.txt -supernova=pihole.2.supernova.txt -eventHorizon=pihole.3.eventHorizon.txt -accretionDisc=pihole.4.accretionDisc.txt -eyeOfTheNeedle=pihole.5.wormhole.txt +basename=pihole +piholeDir=/etc/$basename +adList=$piholeDir/gravity.list blacklist=$piholeDir/blacklist.txt whitelist=$piholeDir/whitelist.txt -latentWhitelist=$origin/latentWhitelist.txt +latentWhitelist=$piholeDir/latentWhitelist.txt +justDomainsExtension=domains +matter=$basename.0.matter.txt +andLight=$basename.1.andLight.txt +supernova=$basename.2.supernova.txt +eventHorizon=$basename.3.eventHorizon.txt +accretionDisc=$basename.4.accretionDisc.txt +eyeOfTheNeedle=$basename.5.wormhole.txt # After setting defaults, check if there's local overrides if [[ -r $piholeDir/pihole.conf ]];then echo "** Local calibration requested..." +<<<<<<< HEAD . $piholeDir/pihole.conf fi echo "** Neutrino emissions detected..." @@ -60,122 +68,212 @@ if [[ -d $piholeDir ]];then else echo "** Creating pihole directory..." $SUDO mkdir $piholeDir +======= + . $piholeDir/pihole.conf +>>>>>>> refs/remotes/jacobsalmela/master fi -# Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and blank lines -for ((i = 0; i < "${#sources[@]}"; i++)) -do - url=${sources[$i]} - # Get just the domain from the URL - domain=$(echo "$url" | cut -d'/' -f3) +########################### +# collapse - begin formation of pihole +function gravity_collapse() { + echo "** Neutrino emissions detected..." - # Save the file as list.#.domain - saveLocation=$origin/list.$i.$domain.$justDomainsExtension + # Create the pihole resource directory if it doesn't exist. Future files will be stored here + if [[ -d $piholeDir ]];then + # Temporary hack to allow non-root access to pihole directory + # Will update later, needed for existing installs, new installs should + # create this directory as non-root + sudo chmod 777 $piholeDir + find "$piholeDir" -type f -exec sudo chmod 666 {} \; + else + echo "** Creating pihole directory..." + mkdir $piholeDir + fi +} - agent="Mozilla/10.0" - - echo -n "Getting $domain list... " - - # Use a case statement to download lists that need special cURL commands - # to complete properly and reset the user agent when required - case "$domain" in - "adblock.mahakala.is") - agent='Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' - cmd="curl -e http://forum.xda-developers.com/" - ;; - - "pgl.yoyo.org") - cmd="curl -d mimetype=plaintext -d hostformat=hosts" - ;; - - # Default is a simple curl request - *) cmd="curl" - esac +# patternCheck - check to see if curl downloaded any new files. +function gravity_patternCheck() { + patternBuffer=$1 + # check if the patternbuffer is a non-zero length file + if [[ -s "$patternBuffer" ]];then + # Some of the blocklists are copyright, they need to be downloaded + # and stored as is. They can be processed for content after they + # have been saved. + cp $patternBuffer $saveLocation + echo "List updated, transport successful..." + else + # curl didn't download any host files, probably because of the date check + echo "No changes detected, transport skipped..." + fi +} +# transport - curl the specified url with any needed command extentions +function gravity_transport() { + url=$1 + cmd_ext=$2 + agent=$3 + # tmp file, so we don't have to store the (long!) lists in RAM patternBuffer=$(mktemp) heisenbergCompensator="" if [[ -r $saveLocation ]]; then + # if domain has been saved, add file for date check to only download newer heisenbergCompensator="-z $saveLocation" fi - CMD="$cmd -s $heisenbergCompensator -A '$agent' $url > $patternBuffer" - $cmd -s $heisenbergCompensator -A "$agent" $url > $patternBuffer - - if [[ -s "$patternBuffer" ]];then - # Remove comments and print only the domain name - # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious - # This helps with that and makes it easier to read - # It also helps with debugging so each stage of the script can be researched more in depth - awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $patternBuffer | \ - sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $saveLocation - echo "Done." - else - echo "Skipping pattern because transporter logic detected no changes..." - fi + # Silently curl url + curl -s $cmd_ext $heisenbergCompensator -A "$agent" $url > $patternBuffer + # Check for list updates + gravity_patternCheck $patternBuffer # Cleanup rm -f $patternBuffer -done - -# Find all files with the .domains extension and compile them into one file and remove CRs -echo "** Aggregating list of domains..." -find $origin/ -type f -name "*.$justDomainsExtension" -exec cat {} \; | tr -d '\r' > $origin/$matter - -# Append blacklist entries if they exist -if [[ -r $blacklist ]];then - numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l) - echo "** Blacklisting $numberOf domain(s)..." - cat $blacklist >> $origin/$matter -fi - -########################### -function gravity_advanced() { - - numberOf=$(wc -l < $origin/$andLight) - echo "** $numberOf domains being pulled in by gravity..." - - # Remove carriage returns and preceding whitespace - # not really needed anymore? - cp $origin/$andLight $origin/$supernova - - # Sort and remove duplicates - sort -u $origin/$supernova > $origin/$eventHorizon - numberOf=$(wc -l < $origin/$eventHorizon) - echo "** $numberOf unique domains trapped in the event horizon." - - # Format domain list as "192.168.x.x domain.com" - echo "** Formatting domains into a HOSTS file..." - cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"' " $0}' > $origin/$accretionDisc - # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it - $SUDO cp $origin/$accretionDisc $adList - kill -HUP $(pidof dnsmasq) } -# Whitelist (if applicable) then remove duplicates and format for dnsmasq -if [[ -r $whitelist ]];then - # Remove whitelist entries - numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) - plural=; [[ "$numberOf" != "1" ]] && plural=s - echo "** Whitelisting $numberOf domain${plural}..." +# spinup - main gravity function +function gravity_spinup() { - # Append a "$" to the end, prepend a "^" to the beginning, and - # replace "." with "\." of each line to turn each entry into a - # regexp so it can be parsed out with grep -x - awk -F '[# \t]' 'NF>0&&$1!="" {print "^"$1"$"}' $whitelist | sed 's/\./\\./g' > $latentWhitelist -else - rm $latentWhitelist -fi + # Loop through domain list. Download each one and remove commented lines (lines beginning with '# 'or '/') and # blank lines + for ((i = 0; i < "${#sources[@]}"; i++)) + do + url=${sources[$i]} + # Get just the domain from the URL + domain=$(echo "$url" | cut -d'/' -f3) -# Prevent our sources from being pulled into the hole -plural=; [[ "${#sources[@]}" != "1" ]] && plural=s -echo "** Whitelisting ${#sources[@]} ad list source${plural}..." -for url in ${sources[@]} -do - echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist -done + # Save the file as list.#.domain + saveLocation=$piholeDir/list.$i.$domain.$justDomainsExtension + activeDomains[$i]=$saveLocation -# Remove whitelist entries from deduped list -grep -vxf $latentWhitelist $origin/$matter > $origin/$andLight + agent="Mozilla/10.0" -gravity_advanced \ No newline at end of file + echo -n " Getting $domain list: " + + # Use a case statement to download lists that need special cURL commands + # to complete properly and reset the user agent when required + case "$domain" in + "adblock.mahakala.is") + agent='Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' + cmd_ext="-e http://forum.xda-developers.com/" + ;; + + "pgl.yoyo.org") + cmd_ext="-d mimetype=plaintext -d hostformat=hosts" + ;; + + # Default is a simple request + *) cmd_ext="" + esac + gravity_transport $url $cmd_ext $agent + done +} + +# Schwarzchild - aggregate domains to one list and add blacklisted domains +function gravity_Schwarzchild() { + + # Find all active domains and compile them into one file and remove CRs + echo "** Aggregating list of domains..." + truncate -s 0 $piholeDir/$matter + for i in "${activeDomains[@]}" + do + cat $i |tr -d '\r' >> $piholeDir/$matter + done +} + +# Pulsar - White/blacklist application +function gravity_pulsar() { + + # Append blacklist entries if they exist + if [[ -r $blacklist ]];then + numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l) + echo "** Blacklisting $numberOf domain(s)..." + cat $blacklist >> $piholeDir/$matter + fi + + # Whitelist (if applicable) domains + if [[ -r $whitelist ]];then + # Remove whitelist entries + numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l) + plural=; [[ "$numberOf" != "1" ]] && plural=s + echo "** Whitelisting $numberOf domain${plural}..." + + # Append a "$" to the end, prepend a "^" to the beginning, and + # replace "." with "\." of each line to turn each entry into a + # regexp so it can be parsed out with grep -x + awk -F '[# \t]' 'NF>0&&$1!="" {print "^"$1"$"}' $whitelist | sed 's/\./\\./g' > $latentWhitelist + else + rm $latentWhitelist + fi + + # Prevent our sources from being pulled into the hole + plural=; [[ "${#sources[@]}" != "1" ]] && plural=s + echo "** Whitelisting ${#sources[@]} ad list source${plural}..." + for url in ${sources[@]} + do + echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist + done + + # Remove whitelist entries from list + grep -vxf $latentWhitelist $piholeDir/$matter > $piholeDir/$andLight +} + +function gravity_unique() { + # Sort and remove duplicates + sort -u $piholeDir/$supernova > $piholeDir/$eventHorizon + numberOf=$(wc -l < $piholeDir/$eventHorizon) + echo "** $numberOf unique domains trapped in the event horizon." +} + +function gravity_hostFormat() { + # Format domain list as "192.168.x.x domain.com" + echo "** Formatting domains into a HOSTS file..." + cat $piholeDir/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"' " $0}' > $piholeDir/$accretionDisc + # Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it +<<<<<<< HEAD + $SUDO cp $origin/$accretionDisc $adList + kill -HUP $(pidof dnsmasq) +======= + cp $piholeDir/$accretionDisc $adList +>>>>>>> refs/remotes/jacobsalmela/master +} + +# blackbody - remove any remnant files from script processes +function gravity_blackbody() { + # Loop through list files + for file in $piholeDir/*.$justDomainsExtension + do + # If list is in active array then leave it (noop) else rm the list + if [[ " ${activeDomains[@]} " =~ " ${file} " ]]; then + : + else + rm -f $file + fi + done +} + +function gravity_advanced() { + # Remove comments and print only the domain name + # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious + # This helps with that and makes it easier to read + # It also helps with debugging so each stage of the script can be researched more in depth + awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $piholeDir/$andLight | \ + sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $piholeDir/$supernova + + numberOf=$(wc -l < $piholeDir/$supernova) + echo "** $numberOf domains being pulled in by gravity..." + + gravity_unique + + sudo kill -HUP $(pidof dnsmasq) +} + +<<<<<<< HEAD +gravity_advanced +======= +gravity_collapse +gravity_spinup +gravity_Schwarzchild +gravity_pulsar +gravity_hostFormat +gravity_advanced +gravity_blackbody +>>>>>>> refs/remotes/jacobsalmela/master