Merge branches 'development' and 'feature/DOReadme' of github.com:pi-hole/pi-hole into feature/DOReadme

# Conflicts:
#	test/centos.Dockerfile
#	test/debian.Dockerfile

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,4 @@
-**By submitting this pull request, I confirm the following (please check boxes, eg [X])Failure to fill the template will close your PR:**
+**By submitting this pull request, I confirm the following (please check boxes, eg [X]) _Failure to fill the template will close your PR_:**
 
 ***Please submit all pull requests against the `development` branch. Failure to do so will delay or deny your request***
 

CONTRIBUTING.md

@@ -33,5 +33,6 @@ When requesting or submitting new features, first consider whether it might be u
 
 - Submit Pull Requests to the **development branch only**.
 - Before Submitting your Pull Request, merge `development` with your new branch and fix any conflicts. (Make sure you don't break anything in development!)
+- Please use the [Google Style Guide for Shell](https://google.github.io/styleguide/shell.xml) for your code submission styles. 
 - Commit Unix line endings.
 - (Optional fun) keep to the theme of Star Trek/black holes/gravity.

README.md

@@ -143,6 +143,7 @@ You can view [real-time stats](http://pi-hole.net/faq/install-the-real-time-lcd-
 -   [Minibian Pi-hole](http://munkjensen.net/wiki/index.php/See_my_Pi-Hole#Minibian_Pi-hole)
 -   [Windows Tray Stat Application](https://github.com/goldbattle/copernicus)
 -   [Let your blink1 device blink when Pi-hole filters ads](https://gist.github.com/elpatron68/ec0b4c582e5abf604885ac1e068d233f)
+-   [Pi-Hole Prometheus exporter](https://github.com/nlamirault/pihole_exporter) : a [Prometheus](https://prometheus.io/) exporter for Pi-Hole
 
 ## Coverage
 

advanced/Scripts/piholeDebug.sh

@@ -27,7 +27,7 @@ PIHOLELOG="/var/log/pihole.log"
 WHITELISTMATCHES="/tmp/whitelistmatches.list"
 
 IPV6_READY=false
-
+TIMEOUT=60
 # Header info and introduction
 cat << EOM
 ::: Beginning Pi-hole debug at $(date)!
@@ -316,6 +316,16 @@ debugLighttpd() {
   echo ":::"
 }
 
+countdown() {
+  tuvix=${TIMEOUT}
+  printf "::: Logging will automatically teminate in ${TIMEOUT} seconds\n"
+  while [ $tuvix -ge 1 ]
+  do
+    printf ":::\t${tuvix} seconds left. \r"
+    sleep 5
+    tuvix=$(( tuvix - 5 ))
+  done
+}
 ### END FUNCTIONS ###
 
 # Gather version of required packages / repositories
@@ -356,10 +366,9 @@ dumpPiHoleLog() {
 	echo -e "::: Try loading a site that you are having trouble with now from a client web browser.. \n:::\t(Press CTRL+C to finish logging.)"
 	header_write "pihole.log"
 	if [ -e "${PIHOLELOG}" ]; then
-		while true; do
+	# Dummy process to use for flagging down tail to terminate
-			tail -f "${PIHOLELOG}" >> ${DEBUG_LOG}
+	  countdown &
-			log_write ""
+		tail -n0 -f --pid=$! "${PIHOLELOG}" >> ${DEBUG_LOG}
-		done
 	else
 		log_write "No pihole.log file found!"
 		printf ":::\tNo pihole.log file found!\n"
@@ -384,10 +393,13 @@ finalWork() {
 	# Check if tricorder.pi-hole.net is reachable and provide token.
 	if [ -n "${tricorder}" ]; then
 		echo "::: Your debug token is : ${tricorder}"
-		echo "::: Please contact the Pi-hole team with your token to being assistance."
+		echo "::: Please contact the Pi-hole team with your token for assistance."
 		echo "::: Thank you."
+	else
+		echo "::: There was an error uploading your debug log."
+		echo "::: Please try again or contact the Pi-hole team for assistance."
 	fi
-		echo "::: Debug log can be found at : /var/log/pihole_debug.log"
+		echo "::: A local copy of the Debug log can be found at : /var/log/pihole_debug.log"
 }
 
 trap finalWork EXIT

automated_install/basic-install.sh

@@ -1149,6 +1149,6 @@ main() {
 	echo "::: The install log is located at: /etc/pihole/install.log"
 }
 
-if [[ -z "$PHTEST" ]] ; then
+if [[ "${PH_TEST}" != true ]] ; then
     main "$@"
 fi

automated_install/uninstall.sh

@@ -82,7 +82,7 @@ removeAndPurge() {
 				read -rp "::: Do you wish to remove ${i} from your system? [y/n]: " yn
 				case ${yn} in
 					[Yy]* ) printf ":::\tRemoving %s..." "${i}"; ${SUDO} ${PKG_REMOVE} "${i}" &> /dev/null & spinner $!; printf "done!\n"; break;;
-					[Nn]* ) printf ":::\tSkipping %s" "${i}\n"; break;;
+					[Nn]* ) printf ":::\tSkipping %s\n" "${i}"; break;;
 					* ) printf "::: You must answer yes or no!\n";;
 				esac
 			done
@@ -154,6 +154,12 @@ removeNoPurge() {
 	${SUDO} rm /usr/local/bin/pihole &> /dev/null
 	${SUDO} rm /etc/bash_completion.d/pihole &> /dev/null
 	${SUDO} rm /etc/sudoers.d/pihole &> /dev/null
+	
+	# If the pihole user exists, then remove
+	if id "pihole" >/dev/null 2>&1; then
+        	echo "::: Removing pihole user..."
+		${SUDO} userdel -r pihole
+	fi
 
 	echo ":::"
 	printf "::: Finished removing PiHole from your system. Sorry to see you go!\n"

gravity.sh

@@ -189,7 +189,7 @@ gravity_Blacklist() {
 	if [[ -f "${blacklistFile}" ]]; then
 	    numBlacklisted=$(wc -l < "${blacklistFile}")
 	    plural=; [[ "$numBlacklisted" != "1" ]] && plural=s
-	    echo -n "::: BlackListing $numBlacklisted domain${plural}..."
+	    echo -n "::: Blacklisting $numBlacklisted domain${plural}..."
 	    cat ${blacklistFile} >> ${piholeDir}/${eventHorizon}
 	    echo " done!"
 	else
@@ -240,7 +240,7 @@ gravity_unique() {
 
 gravity_hostFormat() {
 	# Format domain list as "192.168.x.x domain.com"
-	echo "::: Formatting domains into a HOSTS file..."
+	echo -n "::: Formatting domains into a HOSTS file..."
     # Check vars from setupVars.conf to see if we're using IPv4, IPv6, Or both.
     if [[ -n "${IPV4_ADDRESS}" && -n "${IPV6_ADDRESS}" ]];then
 
@@ -264,6 +264,7 @@ gravity_hostFormat() {
 
 	# Copy the file over as /etc/pihole/gravity.list so dnsmasq can use it
 	cp ${piholeDir}/${accretionDisc} ${adList}
+	echo " done!"
 }
 
 # blackbody - remove any remnant files from script processes

test/centos.Dockerfile

@@ -5,11 +5,12 @@ ENV SCRIPTDIR /opt/pihole
 
 RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
 ADD . $GITDIR
-RUN cp $GITDIR/advanced/Scripts/*.sh \
+RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/
-       $GITDIR/gravity.sh $GITDIR/pihole \
-       $GITDIR/automated_install/*.sh \
-       $SCRIPTDIR/
 ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
 
 RUN true && \
     chmod +x $SCRIPTDIR/*
+
+ENV PH_TEST true
+
+#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/conftest.py

@@ -7,7 +7,22 @@ check_output = testinfra.get_backend(
 
 @pytest.fixture
 def Pihole(Docker):
-    ''' used to contain some script stubbing, now pretty much an alias '''
+    ''' used to contain some script stubbing, now pretty much an alias.
+    Also provides bash as the default run function shell '''
+    def run_bash(self, command, *args, **kwargs):
+        cmd = self.get_command(command, *args)
+        if self.user is not None:
+            out = self.run_local(
+                "docker exec -u %s %s /bin/bash -c %s",
+                self.user, self.name, cmd)
+        else:
+            out = self.run_local(
+                "docker exec %s /bin/bash -c %s", self.name, cmd)
+        out.command = self.encode(cmd)
+        return out
+
+    funcType = type(Docker.run)
+    Docker.run = funcType(run_bash, Docker, testinfra.backend.docker.DockerBackend)
     return Docker
 
 @pytest.fixture

test/debian.Dockerfile

@@ -5,12 +5,12 @@ ENV SCRIPTDIR /opt/pihole
 
 RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole
 ADD . $GITDIR
-RUN cp $GITDIR/advanced/Scripts/*.sh \
+RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/
-       $GITDIR/gravity.sh $GITDIR/pihole \
-       $GITDIR/automated_install/*.sh \
-       $SCRIPTDIR/
 ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR
 
-
 RUN true && \
     chmod +x $SCRIPTDIR/*
+
+ENV PH_TEST true
+
+#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \

test/shellcheck_failing_output.txt

@@ -1,76 +0,0 @@
-============================= test session starts ==============================
-platform linux2 -- Python 2.7.6, pytest-2.9.2, py-1.4.31, pluggy-0.3.1 -- /usr/bin/python
-cachedir: .cache
-rootdir: /home/a/opensource/pi-hole, inifile: 
-plugins: cov-2.3.0, bdd-2.17.0, xdist-1.14, testinfra-1.4.0
-collecting ... collected 7 items
-
-test/test_000_build_containers.py::test_build_pihole_image[test/debian.Dockerfile-pytest_pihole:debian] PASSED
-test/test_000_build_containers.py::test_build_pihole_image[test/centos.Dockerfile-pytest_pihole:centos] PASSED
-test/test_automated_install.py::test_setupVars_are_sourced_to_global_scope[debian] PASSED
-test/test_automated_install.py::test_setupVars_are_sourced_to_global_scope[centos] PASSED
-test/test_automated_install.py::test_setupVars_saved_to_file[debian] PASSED
-test/test_automated_install.py::test_setupVars_saved_to_file[centos] PASSED
-test/test_shellcheck.py::test_scripts_pass_shellcheck FAILED
-
-=================================== FAILURES ===================================
-_________________________ test_scripts_pass_shellcheck _________________________
-
-    def test_scripts_pass_shellcheck():
-        ''' Make sure shellcheck does not find anything wrong with our shell scripts '''
-        shellcheck = "find . -name 'update.sh' | while read file; do shellcheck \"$file\"; done;"
-        results = run_local(shellcheck)
-        print results.stdout
->       assert '' == results.stdout
-E       assert '' == '\nIn ./advanced/Scripts/upda...vent glob interpretation.\n\n'
-E         + 
-E         + In ./advanced/Scripts/update.sh line 24:
-E         + 	while [ "$(ps a | awk '{print $1}' | grep "${pid}")" ]; do
-E         +                 ^-- SC2143: Instead of [ -n $(foo | grep bar) ], use foo | grep -q bar .
-E         + 
-E         + 
-E         + In ./advanced/Scripts/update.sh line 57:
-E         + 	git clone -q --depth 1 "${2}" "${1}" > /dev/null & spinner $!
-E         +                                                                    ^-- SC2086: Double quote to prevent globbing and word splitting.
-E         Detailed information truncated (27 more lines), use "-vv" to show
-
-test/test_shellcheck.py:13: AssertionError
------------------------------ Captured stdout call -----------------------------
-
-In ./advanced/Scripts/update.sh line 24:
-	while [ "$(ps a | awk '{print $1}' | grep "${pid}")" ]; do
-                ^-- SC2143: Instead of [ -n $(foo | grep bar) ], use foo | grep -q bar .
-
-
-In ./advanced/Scripts/update.sh line 57:
-	git clone -q --depth 1 "${2}" "${1}" > /dev/null & spinner $!
-                                                                   ^-- SC2086: Double quote to prevent globbing and word splitting.
-
-
-In ./advanced/Scripts/update.sh line 65:
-	git stash -q > /dev/null & spinner $!
-                                           ^-- SC2086: Double quote to prevent globbing and word splitting.
-
-
-In ./advanced/Scripts/update.sh line 66:
-	git pull -q > /dev/null & spinner $!
-                                          ^-- SC2086: Double quote to prevent globbing and word splitting.
-
-
-In ./advanced/Scripts/update.sh line 107:
-if [[ ${piholeVersion} == ${piholeVersionLatest} && ${webVersion} == ${webVersionLatest} ]]; then
-                          ^-- SC2053: Quote the rhs of = in [[ ]] to prevent glob interpretation.
-                                                                     ^-- SC2053: Quote the rhs of = in [[ ]] to prevent glob interpretation.
-
-
-In ./advanced/Scripts/update.sh line 112:
-elif [[ ${piholeVersion} == ${piholeVersionLatest} && ${webVersion} != ${webVersionLatest} ]]; then
-                            ^-- SC2053: Quote the rhs of = in [[ ]] to prevent glob interpretation.
-
-
-In ./advanced/Scripts/update.sh line 120:
-elif [[ ${piholeVersion} != ${piholeVersionLatest} && ${webVersion} == ${webVersionLatest} ]]; then
-                                                                       ^-- SC2053: Quote the rhs of = in [[ ]] to prevent glob interpretation.
-
-
-===================== 1 failed, 6 passed in 24.01 seconds ======================

test/test_automated_install.py

@@ -10,8 +10,9 @@ SETUPVARS = {
 }
 
 def test_setupVars_are_sourced_to_global_scope(Pihole):
-    ''' currently update_dialogs sources setupVars with a dot, 
+    ''' currently update_dialogs sources setupVars with a dot,
-    then various other functions use the variables '''
+    then various other functions use the variables.
+    This confirms the sourced variables are in scope between functions '''
     setup_var_file = 'cat <<EOF> /etc/pihole/setupVars.conf\n'
     for k,v in SETUPVARS.iteritems():
         setup_var_file += "{}={}\n".format(k, v)
@@ -19,15 +20,15 @@ def test_setupVars_are_sourced_to_global_scope(Pihole):
     Pihole.run(setup_var_file)
 
     script = dedent('''\
-    #!/bin/bash -e
+    set -e
     printSetupVars() {
         # Currently debug test function only
         echo "Outputting sourced variables"
-        echo "PIHOLE_INTERFACE=\${PIHOLE_INTERFACE}"
+        echo "PIHOLE_INTERFACE=${PIHOLE_INTERFACE}"
-        echo "IPV4_ADDRESS=\${IPV4_ADDRESS}"
+        echo "IPV4_ADDRESS=${IPV4_ADDRESS}"
-        echo "IPV6_ADDRESS=\${IPV6_ADDRESS}"
+        echo "IPV6_ADDRESS=${IPV6_ADDRESS}"
-        echo "PIHOLE_DNS_1=\${PIHOLE_DNS_1}"
+        echo "PIHOLE_DNS_1=${PIHOLE_DNS_1}"
-        echo "PIHOLE_DNS_2=\${PIHOLE_DNS_2}"
+        echo "PIHOLE_DNS_2=${PIHOLE_DNS_2}"
     }
     update_dialogs() {
         . /etc/pihole/setupVars.conf
@@ -49,10 +50,9 @@ def test_setupVars_saved_to_file(Pihole):
     Pihole.run(set_setup_vars).stdout
 
     script = dedent('''\
-    #!/bin/bash -e
+    set -e
     echo start
     TERM=xterm
-    PHTEST=TRUE
     source /opt/pihole/basic-install.sh
     {}
     finalExports
@@ -64,14 +64,39 @@ def test_setupVars_saved_to_file(Pihole):
     for k,v in SETUPVARS.iteritems():
         assert "{}={}".format(k, v) in output
 
-def run_script(Pihole, script, file="/test.sh"):
+def test_configureFirewall_firewalld_no_errors(Pihole):
-    _write_test_script(Pihole, script, file=file)
+    ''' confirms firewalld rules are applied when appopriate '''
-    result = Pihole.run(file)
+    mock_command('firewall-cmd', '0', Pihole)
+    configureFirewall = Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    configureFirewall
+    ''')
+    expected_stdout = '::: Configuring firewalld for httpd and dnsmasq.'
+    assert expected_stdout in configureFirewall.stdout
+    firewall_calls = Pihole.run('cat /var/log/firewall-cmd').stdout
+    assert 'firewall-cmd --state' in firewall_calls
+    assert 'firewall-cmd --permanent --add-port=80/tcp' in firewall_calls
+    assert 'firewall-cmd --permanent --add-port=53/tcp' in firewall_calls
+    assert 'firewall-cmd --permanent --add-port=53/udp' in firewall_calls
+    assert 'firewall-cmd --reload' in firewall_calls
+
+
+# Helper functions
+def mock_command(script, result, container):
+    ''' Allows for setup of commands we don't really want to have to run for real in unit tests '''
+    ''' TODO: support array of results that enable the results to change over multiple executions of a command '''
+    full_script_path = '/usr/local/bin/{}'.format(script)
+    mock_script = dedent('''\
+    #!/bin/bash -e
+    echo "\$0 \$@" >> /var/log/{script}
+    exit {retcode}
+    '''.format(script=script, retcode=result))
+    container.run('''
+    cat <<EOF> {script}\n{content}\nEOF
+    chmod +x {script}
+    '''.format(script=full_script_path, content=mock_script))
+
+def run_script(Pihole, script):
+    result = Pihole.run(script)
     assert result.rc == 0
     return result
-
-def _write_test_script(Pihole, script, file):
-    ''' Running the test script blocks directly can behave differently with regard to global vars '''
-    ''' this is a cheap work around to that until all functions no longer rely on global variables '''
-    Pihole.run('cat <<EOF> {file}\n{script}\nEOF'.format(file=file, script=script))
-    Pihole.run('chmod +x {}'.format(file))