Merge remote-tracking branch 'refs/remotes/origin/master' into patch-1

README.md

@@ -60,4 +60,7 @@ A technical and detailed description can be found [here](http://jacobsalmela.com
 ## Other Operating Systems
 This script will work for other UNIX-like systems with some slight **modifications**.  As long as you can install `dnsmasq` and a Webserver, it should work OK.  The automated install only works for a clean install of Raspiban right now since that is how the project originated.
 
+### Examples Of The Pi-hole On Other Operating Systems
+- [Sky-Hole](http://dlaa.me/blog/post/skyhole)
+
 [![Donate](https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif "AdminLTE Presentation")](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=3J2L3Z4DHW9UY "Donate")

advanced/Scripts/piholeLogFlush.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+# Flushes /var/log/pihole.log
+truncate -s 0 /var/log/pihole.log

advanced/Scripts/whitelist.sh

@@ -1,13 +1,44 @@
 #!/bin/bash
+
+if [ $# = 0 ]; then
+    echo "Immediately whitelists one or more domains."
+    echo "Usage: whitelist.sh domain1 [domain2 ...]"
+fi
+
+combopattern=""
+
 # For each argument passed to this script
 for var in "$@"
 do
   echo "Whitelisting $var..."
-        # Use sed to search for the domain in /etc/pihole/gravity.list and remove it using an in-place edit
+
-        sed -i "/$var/d" /etc/pihole/gravity.list
+  # Construct basic pattern to match domain name.
-        # Also add the domain to the whitelist.txt in /etc/pihole
+  basicpattern=$(echo $var | awk -F '[# \t]' 'NF>0&&$1!="" {print ""$1""}' | sed 's/\./\\./g')
-        echo "$var" >> /etc/pihole/whitelist.txt
+
+  if [ "$basicpattern" != "" ]; then
+    # Add to the combination pattern that will be used below
+    if [ "$combopattern" != "" ]; then combopattern="$combopattern|"; fi
+    combopattern="$combopattern$basicpattern"
+
+    # Also add the domain to the whitelist but only if it's not already present
+    grep -E -q "^$basicpattern$" /etc/pihole/whitelist.txt \
+    || echo "$var" >> /etc/pihole/whitelist.txt
+  fi
 done
+
+# Now report on and remove matched domains
+if [ "$combopattern" != "" ]; then
+  echo "Modifying hosts file..."
+  
+  # Construct pattern to match entry in hosts file.
+  # This consists of one or more IP addresses followed by the domain name.
+  pattern=$(echo $combopattern | awk -F '[# \t]' '{printf "%s", "^(([0-9]+\.){3}[0-9]+ +)+("$1")$"}')
+
+  # Output what will be removed and then actually remove
+  sed -r -n 's/'"$pattern"'/  Removed: \3/p' /etc/pihole/gravity.list
+  sed -r -i '/'"$pattern"'/d' /etc/pihole/gravity.list
+
   echo "** $# domain(s) whitelisted."
   # Force dnsmasq to reload /etc/pihole/gravity.list
   kill -HUP $(pidof dnsmasq)
+fi

advanced/pihole.cron

@@ -1 +1,26 @@
-@weekly sudo /usr/local/bin/gravity.sh
+# /etc/crontab: system-wide crontab
+# Unlike any other crontab you don't have to run the `crontab'
+# command to install the new version when you edit this file
+# and files in /etc/cron.d. These files also have username fields,
+# that none of the other crontabs do.
+
+SHELL=/bin/sh
+PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
+
+# m h dom mon dow user  command
+17 *    * * *   root    cd / && run-parts --report /etc/cron.hourly
+25 6    * * *   root    test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.daily )
+47 6    * * 7   root    test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.weekly )
+52 6    1 * *   root    test -x /usr/sbin/anacron || ( cd / && run-parts --report /etc/cron.monthly )
+
+# Pi-hole: Update the ad sources once a week on Sunday at 01:59
+#          Download any updates from the ad lists
+59 1    * * 7   root    /usr/local/bin/gravity.sh
+
+# Pi-hole: Parse the log file before it is flushed and save the stats to a database
+#          This will be used for a historical view of your Pi-hole's performance
+#50 11  * * *   root    /usr/local/bin/dailyLog.sh
+
+# Pi-hole: Flush the log daily at 11:58 so it doesn't get out of control
+#          Stats will be viewable in the Web interface thanks to the cron job above
+58 11   * * *   root    /usr/local/bin/piholeLogFlush.sh

gravity.sh

@@ -19,7 +19,7 @@ fi
 sources=('https://adaway.org/hosts.txt'
 'http://adblock.gjtech.net/?format=unix-hosts'
 #'http://adblock.mahakala.is/'
-'http://hosts-file.net/.%5Cad_servers.txt'
+'http://hosts-file.net/ad_servers.txt'
 'http://www.malwaredomainlist.com/hostslist/hosts.txt'
 'http://pgl.yoyo.org/adservers/serverlist.php?'
 'http://someonewhocares.org/hosts/hosts'
@@ -37,7 +37,6 @@ eventHorizon=pihole.3.eventHorizon.txt
 accretionDisc=pihole.4.accretionDisc.txt
 eyeOfTheNeedle=pihole.5.wormhole.txt
 blacklist=$piholeDir/blacklist.txt
-latentBlacklist=$origin/latentBlacklist.txt
 whitelist=$piholeDir/whitelist.txt
 latentWhitelist=$origin/latentWhitelist.txt
 
@@ -46,8 +45,6 @@ if [[ -r $piholeDir/pihole.conf ]];then
     echo "** Local calibration requested..."
 	. $piholeDir/pihole.conf
 fi
-
-
 echo "** Neutrino emissions detected..."
 
 # Create the pihole resource directory if it doesn't exist.  Future files will be stored here
@@ -58,35 +55,6 @@ else
 	$SUDO mkdir $piholeDir
 fi
 
-# Add additional swap to prevent the "Error fork: unable to allocate memory" message:  https://github.com/jacobsalmela/pi-hole/issues/37
-function createSwapFile()
-#########################
-	{
-	echo "** Creating more swap space to accomodate large solar masses..."
-	$SUDO dphys-swapfile swapoff
-	$SUDO curl -s -o /etc/dphys-swapfile https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/dphys-swapfile
-	$SUDO dphys-swapfile setup
-	$SUDO dphys-swapfile swapon
-	}
-
-
-if [[ -n "$noSwap" ]]; then
-    # if $noSwap is set, don't do anything
-    :
-elif [[ -f /etc/dphys-swapfile ]];then
-	swapSize=$(cat /etc/dphys-swapfile | grep -m1 CONF_SWAPSIZE | cut -d'=' -f2)
-	if [[ $swapSize != 500 ]];then
-		mv /etc/dphys-swapfile /etc/dphys-swapfile.orig
-		echo "** Current swap size is $swapSize"
-		createSwapFile
-	else
-		:
-	fi
-else
-	echo "** No swap file found.  Creating one..."
-	createSwapFile
-fi
-
 # Loop through domain list.  Download each one and remove commented lines (lines beginning with '# 'or '/') and blank lines
 for ((i = 0; i < "${#sources[@]}"; i++))
 do
@@ -97,27 +65,50 @@ do
 	# Save the file as list.#.domain
 	saveLocation=$origin/list.$i.$domain.$justDomainsExtension
 
+	agent="Mozilla/10.0"
+
 	echo -n "Getting $domain list... "
-	# Use a case statement to download lists that need special cURL commands to complete properly
+
+	# Use a case statement to download lists that need special cURL commands
+	# to complete properly and reset the user agent when required
 	case "$domain" in
-		"adblock.mahakala.is") data=$(curl -s -A 'Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0' -e http://forum.xda-developers.com/ -z $saveLocation $url);;
+		"adblock.mahakala.is")
+			agent='Mozilla/5.0 (X11; Linux x86_64; rv:30.0) Gecko/20100101 Firefox/30.0'
+			cmd="curl -e http://forum.xda-developers.com/"
+			;;
 
-		"pgl.yoyo.org") data=$(curl -s -d mimetype=plaintext -d hostformat=hosts -z $saveLocation $url);;
+		"pgl.yoyo.org")
+			cmd="curl -d mimetype=plaintext -d hostformat=hosts"
+			;;
 
-		*) data=$(curl -s -z $saveLocation -A "Mozilla/10.0" $url);;
+		# Default is a simple curl request
+		*) cmd="curl"
 	esac
 
-	if [[ -n "$data" ]];then
+	# tmp file, so we don't have to store the (long!) lists in RAM
+	patternBuffer=$(mktemp)
+	heisenbergCompensator=""
+	if [[ -r $saveLocation ]]; then
+		heisenbergCompensator="-z $saveLocation"
+	fi
+	CMD="$cmd -s $heisenbergCompensator -A '$agent' $url > $patternBuffer"
+	$cmd -s $heisenbergCompensator -A "$agent" $url > $patternBuffer
+
+
+	if [[ -s "$patternBuffer" ]];then
 		# Remove comments and print only the domain name
 		# Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious
 		# This helps with that and makes it easier to read
 		# It also helps with debugging so each stage of the script can be researched more in depth
-		echo "$data" | awk 'NF {if ($1 !~ "#") { if (NF>1) {print $2} else {print $1}}}' | \
+		awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $patternBuffer | \
-			sed -e 's/^[. \t]*//' -e 's/\.\.\+/./g' -e 's/[. \t]*$//' | grep "\." > $saveLocation
+			sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $saveLocation
 		echo "Done."
 	else
-		echo "Skipping list because it does not have any new entries."
+		echo "Skipping pattern because transporter logic detected no changes..."
 	fi
+
+	# Cleanup
+	rm -f $patternBuffer
 done
 
 # Find all files with the .domains extension and compile them into one file and remove CRs
@@ -125,39 +116,43 @@ echo "** Aggregating list of domains..."
 find $origin/ -type f -name "*.$justDomainsExtension" -exec cat {} \; | tr -d '\r' > $origin/$matter
 
 # Append blacklist entries if they exist
-if [[ -f $blacklist ]];then
+if [[ -r $blacklist ]];then
 	numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l)
 	echo "** Blacklisting $numberOf domain(s)..."
 	cat $blacklist >> $origin/$matter
-else
-	:
 fi
 
-function gravity_advanced()
 ###########################
-	{
+function gravity_advanced() {
-	numberOf=$(cat $origin/$andLight | sed '/^\s*$/d' | wc -l)
+
+	numberOf=$(wc -l < $origin/$andLight)
 	echo "** $numberOf domains being pulled in by gravity..."
+
 	# Remove carriage returns and preceding whitespace
-	cat $origin/$andLight | sed $'s/\r$//' | sed '/^\s*$/d' > $origin/$supernova
+	# not really needed anymore?
+	cp $origin/$andLight $origin/$supernova
+
 	# Sort and remove duplicates
-	cat $origin/$supernova | sort | uniq > $origin/$eventHorizon
+	sort -u  $origin/$supernova > $origin/$eventHorizon
-	numberOf=$(cat $origin/$eventHorizon | sed '/^\s*$/d' | wc -l)
+	numberOf=$(wc -l < $origin/$eventHorizon)
 	echo "** $numberOf unique domains trapped in the event horizon."
+
 	# Format domain list as "192.168.x.x domain.com"
 	echo "** Formatting domains into a HOSTS file..."
-	cat $origin/$eventHorizon | awk '{sub(/\r$/,""); print "'"$piholeIP"'" $0}' > $origin/$accretionDisc
+	awk '{print "'"$piholeIP"'" $1}' $origin/$eventHorizon > $origin/$accretionDisc
+
 	# Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it
 	$SUDO cp $origin/$accretionDisc $adList
 	kill -HUP $(pidof dnsmasq)
 }
 
 # Whitelist (if applicable) then remove duplicates and format for dnsmasq
-if [[ -f $whitelist ]];then
+if [[ -r $whitelist ]];then
 	# Remove whitelist entries
 	numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l)
 	plural=; [[ "$numberOf" != "1" ]] && plural=s
 	echo "** Whitelisting $numberOf domain${plural}..."
+
 	# Append a "$" to the end, prepend a "^" to the beginning, and
 	# replace "." with "\." of each line to turn each entry into a
 	# regexp so it can be parsed out with grep -x
@@ -174,6 +169,7 @@ do
 	echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist
 done
 
+# Remove whitelist entries from deduped list
 grep -vxf $latentWhitelist $origin/$matter > $origin/$andLight
 
 gravity_advanced