Merge branch 'development' into new/gravitydb

Signed-off-by: DL6ER <dl6er@dl6er.de>

README.md

@@ -17,9 +17,9 @@ The Pi-hole[®](https://pi-hole.net/trademark-rules-and-brand-guidelines/) is a
 - **Free**: open source software which helps ensure _you_ are the sole person in control of your privacy
 
 -----
-<a href="https://www.codacy.com/app/Pi-hole/pi-hole?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=pi-hole/pi-hole&amp;utm_campaign=Badge_Grade"><img src="https://api.codacy.com/project/badge/Grade/c558a0f8d7124c99b02b84f0f5564238" alt="Codacy Grade"/></a>
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/c558a0f8d7124c99b02b84f0f5564238)](https://www.codacy.com/app/Pi-hole/pi-hole?utm_source=github.com&amp;utm_medium=referral&amp;utm_content=pi-hole/pi-hole&amp;utm_campaign=Badge_Grade)
-<a href="https://travis-ci.org/pi-hole/pi-hole"><img src="https://travis-ci.org/pi-hole/pi-hole.svg?branch=development" alt="Travis Build Status"/></a>
+[![Build Status](https://travis-ci.org/pi-hole/pi-hole.svg?branch=development)](https://travis-ci.org/pi-hole/pi-hole)
-<a href="https://www.bountysource.com/trackers/3011939-pi-hole-pi-hole?utm_source=3011939&utm_medium=shield&utm_campaign=TRACKER_BADGE"><img src="https://www.bountysource.com/badge/tracker?tracker_id=3011939" alt="BountySource"/></a>
+[![BountySource](https://www.bountysource.com/badge/tracker?tracker_id=3011939)](https://www.bountysource.com/trackers/3011939-pi-hole-pi-hole?utm_source=3011939&utm_medium=shield&utm_campaign=TRACKER_BADGE)
 
 ## One-Step Automated Install
 Those who want to get started quickly and conveniently may install Pi-hole using the following command:

advanced/Scripts/piholeDebug.sh

@@ -109,7 +109,6 @@ FTL_PORT="${RUN_DIRECTORY}/pihole-FTL.port"
 PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log"
 PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*"
 PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log"
-PIHOLE_DEBUG_LOG_SANITIZED="${LOG_DIRECTORY}/pihole_debug-sanitized.log"
 PIHOLE_FTL_LOG="${LOG_DIRECTORY}/pihole-FTL.log"
 
 PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access.log"
@@ -209,11 +208,6 @@ log_write() {
 copy_to_debug_log() {
     # Copy the contents of file descriptor 3 into the debug log
     cat /proc/$$/fd/3 > "${PIHOLE_DEBUG_LOG}"
-    # Since we use color codes such as '\e[1;33m', they should be removed before being
-    # uploaded to our server, since it can't properly display in color
-    # This is accomplished by use sed to remove characters matching that patter
-    # The entire file is then copied over to a sanitized version of the log
-    sed 's/\[[0-9;]\{1,5\}m//g' > "${PIHOLE_DEBUG_LOG_SANITIZED}" <<< cat "${PIHOLE_DEBUG_LOG}"
 }
 
 initialize_debug() {
@@ -1147,20 +1141,20 @@ analyze_pihole_log() {
     IFS="$OLD_IFS"
 }
 
-tricorder_use_nc_or_ssl() {
+tricorder_use_nc_or_curl() {
-    # Users can submit their debug logs using nc (unencrypted) or openssl (enrypted) if available
+    # Users can submit their debug logs using nc (unencrypted) or curl (encrypted) if available
-    # Check for openssl first since encryption is a good thing
+    # Check for curl first since encryption is a good thing
-    if command -v openssl &> /dev/null; then
+    if command -v curl &> /dev/null; then
         # If the command exists,
-        log_write "    * Using ${COL_GREEN}openssl${COL_NC} for transmission."
+        log_write "    * Using ${COL_GREEN}curl${COL_NC} for transmission."
-        # encrypt and transmit the log and store the token returned in a variable
+        # transmit he log via TLS and store the token returned in a variable
-        tricorder_token=$(< ${PIHOLE_DEBUG_LOG_SANITIZED} openssl s_client -quiet -connect tricorder.pi-hole.net:${TRICORDER_SSL_PORT_NUMBER} 2> /dev/null)
+        tricorder_token=$(curl --silent --upload-file ${PIHOLE_DEBUG_LOG} https://tricorder.pi-hole.net:${TRICORDER_SSL_PORT_NUMBER})
     # Otherwise,
     else
         # use net cat
         log_write "${INFO} Using ${COL_YELLOW}netcat${COL_NC} for transmission."
         # Save the token returned by our server in a variable
-        tricorder_token=$(< ${PIHOLE_DEBUG_LOG_SANITIZED} nc tricorder.pi-hole.net ${TRICORDER_NC_PORT_NUMBER})
+        tricorder_token=$(< ${PIHOLE_DEBUG_LOG} nc tricorder.pi-hole.net ${TRICORDER_NC_PORT_NUMBER})
     fi
 }
 
@@ -1222,7 +1216,7 @@ upload_to_tricorder() {
         log_write "   * Please try again or contact the Pi-hole team for assistance."
     fi
     # Finally, show where the log file is no matter the outcome of the function so users can look at it
-    log_write "   * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG_SANITIZED}${COL_NC}\\n"
+    log_write "   * A local copy of the debug log can be found at: ${COL_CYAN}${PIHOLE_DEBUG_LOG}${COL_NC}\\n"
 }
 
 # Run through all the functions we made

advanced/Scripts/query.sh

@@ -54,7 +54,7 @@ scanList(){
     # /dev/null forces filename to be printed when only one list has been generated
     # shellcheck disable=SC2086
     case "${type}" in
-        "exact" ) grep -i -E -l "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
+        "exact" ) grep -i -E -l "(^|(?<!#)\\s)${domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
         "wc"    ) grep -i -o -m 1 "/${domain}/" ${lists} 2>/dev/null;;
         *       ) grep -i "${domain}" ${lists} /dev/null 2>/dev/null;;
     esac

advanced/Scripts/update.sh

@@ -146,6 +146,20 @@ main() {
         FTL_update=false
     fi
 
+    # Determine FTL branch
+    local ftlBranch
+    if [[ -f "/etc/pihole/ftlbranch" ]]; then
+        ftlBranch=$(</etc/pihole/ftlbranch)
+    else
+        ftlBranch="master"
+    fi
+
+    if [[ ! "${ftlBranch}" == "master" && ! "${ftlBranch}" == "development" ]]; then
+        # Notify user that they are on a custom branch which might mean they they are lost
+        # behind if a branch was merged to development and got abandoned
+        printf "  %b %bWarning:%b You are using FTL from a custom branch (%s) and might be missing future releases.\\n" "${INFO}" "${COL_LIGHT_RED}" "${COL_NC}" "${ftlBranch}"
+    fi
+
     if [[ "${core_update}" == false && "${web_update}" == false && "${FTL_update}" == false ]]; then
         echo ""
         echo -e "  ${TICK} Everything is up to date!"

advanced/index.php

@@ -40,13 +40,6 @@ $validExtTypes = array("asp", "htm", "html", "php", "rss", "xml", "");
 // Get extension of current URL
 $currentUrlExt = pathinfo($_SERVER["REQUEST_URI"], PATHINFO_EXTENSION);
 
-// Check if this is served over HTTP or HTTPS
-if(isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == "on") {
-    $proto = "https";
-} else {
-    $proto = "http";
-}
-
 // Set mobile friendly viewport
 $viewPort = '<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>';
 
@@ -229,10 +222,10 @@ setHeader();
   <?=$viewPort ?>
   <meta name="robots" content="noindex,nofollow"/>
   <meta http-equiv="x-dns-prefetch-control" content="off">
-  <link rel="shortcut icon" href="<?=$proto ?>://pi.hole/admin/img/favicon.png" type="image/x-icon"/>
+  <link rel="shortcut icon" href="//pi.hole/admin/img/favicon.png" type="image/x-icon"/>
-  <link rel="stylesheet" href="<?=$proto ?>://pi.hole/pihole/blockingpage.css" type="text/css"/>
+  <link rel="stylesheet" href="//pi.hole/pihole/blockingpage.css" type="text/css"/>
   <title>● <?=$serverName ?></title>
-  <script src="<?=$proto ?>://pi.hole/admin/scripts/vendor/jquery.min.js"></script>
+  <script src="//pi.hole/admin/scripts/vendor/jquery.min.js"></script>
   <script>
     window.onload = function () {
       <?php

automated install/basic-install.sh

@@ -1094,7 +1094,7 @@ setPrivacyLevel() {
     local LevelCommand
     local LevelOptions
 
-    LevelCommand=(whiptail --separate-output --radiolist "Select a privacy mode for FTL." "${r}" "${c}" 6)
+    LevelCommand=(whiptail --separate-output --radiolist "Select a privacy mode for FTL. https://docs.pi-hole.net/ftldns/privacylevels/" "${r}" "${c}" 6)
 
     # The default selection is level 0
     LevelOptions=(
@@ -2294,10 +2294,12 @@ get_binary_name() {
     elif [[ "${machine}" == "x86_64" ]]; then
         # This gives the architecture of packages dpkg installs (for example, "i386")
         local dpkgarch
-        dpkgarch=$(dpkg --print-architecture 2> /dev/null)
+        dpkgarch=$(dpkg --print-architecture 2> /dev/null || true)
 
         # Special case: This is a 32 bit OS, installed on a 64 bit machine
         # -> change machine architecture to download the 32 bit executable
+        # We only check this for Debian-based systems as this has been an issue
+        # in the past (see https://github.com/pi-hole/pi-hole/pull/2004)
         if [[ "${dpkgarch}" == "i386" ]]; then
             printf "%b  %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}"
             binary="pihole-FTL-linux-x86_32"

automated install/uninstall.sh

@@ -131,8 +131,7 @@ removeNoPurge() {
         echo -e "  ${TICK} Removed /etc/cron.d/pihole"
     fi
 
-    package_check lighttpd > /dev/null
+    if package_check lighttpd > /dev/null; then
-    if [[ $? -eq 1 ]]; then
         ${SUDO} rm -rf /etc/lighttpd/ &> /dev/null
         echo -e "  ${TICK} Removed lighttpd"
     else

requirements.txt

@@ -1,6 +1,6 @@
-docker-compose
+docker-compose==1.23.2
-pytest
+pytest==4.3.0
-pytest-xdist
+pytest-xdist==1.26.1
-pytest-cov
+pytest-cov==2.6.1
-testinfra
+testinfra==1.19.0
-tox
+tox==3.7.0