Merge remote-tracking branch 'origin/development' into new/regex-lists

advanced/Scripts/piholeCheckout.sh

@@ -166,6 +166,9 @@ checkout() {
     if check_download_exists "$path"; then
         echo "  ${TICK} Branch ${2} exists"
         echo "${2}" > /etc/pihole/ftlbranch
+        FTLinstall "${binary}"
+        start_service pihole-FTL
+        enable_service pihole-FTL
     else
         echo "  ${CROSS} Requested branch \"${2}\" is not available"
         ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') )
@@ -180,7 +183,7 @@ checkout() {
   fi
 
   # Force updating everything
-  if [[  ! "${1}" == "web"  ]]; then
+  if [[  ! "${1}" == "web" && ! "${1}" == "ftl" ]]; then
     echo -e "  ${INFO} Running installer to upgrade your installation"
     if "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" --unattended; then
       exit 0

advanced/Scripts/piholeDebug.sh

@@ -659,15 +659,22 @@ check_required_ports() {
   # Sort the addresses and remove duplicates
   while IFS= read -r line; do
       ports_in_use+=( "$line" )
-  done < <( lsof -i -P -n | awk -F' ' '/LISTEN/ {print $9, $1}' | sort -n | uniq | cut -d':' -f2 )
+  done < <( lsof -iTCP -sTCP:LISTEN -P -n +c 10 )
 
   # Now that we have the values stored,
   for i in "${!ports_in_use[@]}"; do
     # loop through them and assign some local variables
-    local port_number
-    port_number="$(echo "${ports_in_use[$i]}" | awk '{print $1}')"
     local service_name
-    service_name=$(echo "${ports_in_use[$i]}" | awk '{print $2}')
+    service_name=$(echo "${ports_in_use[$i]}" | awk '{print $1}')
+    local protocol_type
+    protocol_type=$(echo "${ports_in_use[$i]}" | awk '{print $5}')
+    local port_number
+    port_number="$(echo "${ports_in_use[$i]}" | awk '{print $9}')"
+
+    # Skip the line if it's the titles of the columns the lsof command produces
+    if [[ "${service_name}" == COMMAND ]]; then
+      continue
+    fi
     # Use a case statement to determine if the right services are using the right ports
     case "${port_number}" in
       53) compare_port_to_service_assigned  "${resolver}"
@@ -677,7 +684,7 @@ check_required_ports() {
       4711) compare_port_to_service_assigned  "${ftl}"
           ;;
       # If it's not a default port that Pi-hole needs, just print it out for the user to see
-      *) log_write "[${port_number}] is in use by ${service_name}";
+      *) log_write "${port_number} ${service_name} (${protocol_type})";
     esac
   done
 }
@@ -773,7 +780,8 @@ dig_at() {
   # Find a random blocked url that has not been whitelisted.
   # This helps emulate queries to different domains that a user might query
   # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains
-  local random_url=$(shuf -n 1 "${PIHOLE_BLOCKLIST_FILE}" | awk -F ' ' '{ print $2 }')
+  local random_url
+  random_url=$(shuf -n 1 "${PIHOLE_BLOCKLIST_FILE}")
 
   # First, do a dig on localhost to see if Pi-hole can use itself to block a domain
   if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then

advanced/Scripts/query.sh

@@ -0,0 +1,222 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC1090
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2018 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# Query Domain Lists
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+# Globals
+piholeDir="/etc/pihole"
+adListsList="$piholeDir/adlists.list"
+wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf"
+options="$*"
+adlist=""
+all=""
+exact=""
+blockpage=""
+matchType="match"
+
+colfile="/opt/pihole/COL_TABLE"
+source "${colfile}"
+
+# Scan an array of files for matching strings
+scanList(){
+  # Escape full stops
+  local domain="${1//./\\.}" lists="${2}" type="${3:-}"
+
+  # Prevent grep from printing file path
+  cd "$piholeDir" || exit 1
+
+  # Prevent grep -i matching slowly: http://bit.ly/2xFXtUX
+  export LC_CTYPE=C
+
+  # /dev/null forces filename to be printed when only one list has been generated
+  # shellcheck disable=SC2086
+  case "${type}" in
+    "exact" ) grep -i -E -l "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
+    "wc"    ) grep -i -o -m 1 "/${domain}/" ${lists} 2>/dev/null;;
+    *       ) grep -i "${domain}" ${lists} /dev/null 2>/dev/null;;
+  esac
+}
+
+if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then
+  echo "Usage: pihole -q [option] <domain>
+Example: 'pihole -q -exact domain.com'
+Query the adlists for a specified domain
+
+Options:
+  -adlist             Print the name of the block list URL
+  -exact              Search the block lists for exact domain matches
+  -all                Return all query matches within a block list
+  -h, --help          Show this help dialog"
+  exit 0
+fi
+
+if [[ ! -e "$adListsList" ]]; then
+  echo -e "${COL_LIGHT_RED}The file $adListsList was not found${COL_NC}"
+  exit 1
+fi
+
+# Handle valid options
+if [[ "${options}" == *"-bp"* ]]; then
+  exact="exact"; blockpage=true
+else
+  [[ "${options}" == *"-adlist"* ]] && adlist=true
+  [[ "${options}" == *"-all"* ]] && all=true
+  if [[ "${options}" == *"-exact"* ]]; then
+    exact="exact"; matchType="exact ${matchType}"
+  fi
+fi
+
+# Strip valid options, leaving only the domain and invalid options
+# This allows users to place the options before or after the domain
+options=$(sed -E 's/ ?-(bp|adlists?|all|exact) ?//g' <<< "${options}")
+
+# Handle remaining options
+# If $options contain non ASCII characters, convert to punycode
+case "${options}" in
+  ""             ) str="No domain specified";;
+  *" "*          ) str="Unknown query option specified";;
+  *[![:ascii:]]* ) domainQuery=$(idn2 "${options}");;
+  *              ) domainQuery="${options}";;
+esac
+
+if [[ -n "${str:-}" ]]; then
+  echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information."
+  exit 1
+fi
+
+# Scan Whitelist and Blacklist
+lists="whitelist.txt blacklist.txt"
+mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")"
+ if [[ -n "${results[*]}" ]]; then
+  wbMatch=true
+   # Loop through each result in order to print unique file title once
+  for result in "${results[@]}"; do
+    fileName="${result%%.*}"
+     if [[ -n "${blockpage}" ]]; then
+      echo "π ${result}"
+      exit 0
+    elif [[ -n "${exact}" ]]; then
+      echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
+    else
+      # Only print filename title once per file
+      if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then
+        echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
+        fileName_prev="${fileName}"
+      fi
+      echo "   ${result#*:}"
+    fi
+  done
+fi
+
+# Scan Wildcards
+if [[ -e "${wildcardlist}" ]]; then
+  # Determine all subdomains, domain and TLDs
+  mapfile -t wildcards <<< "$(processWildcards "${domainQuery}")"
+   for match in "${wildcards[@]}"; do
+    # Search wildcard list for matches
+    mapfile -t results <<< "$(scanList "${match}" "${wildcardlist}" "wc")"
+     if [[ -n "${results[*]}" ]]; then
+      if [[ -z "${wcMatch:-}" ]] && [[ -z "${blockpage}" ]]; then
+        wcMatch=true
+        echo " ${matchType^} found in ${COL_BOLD}Wildcards${COL_NC}:"
+      fi
+       case "${blockpage}" in
+        true ) echo "π ${wildcardlist##*/}"; exit 0;;
+        *    ) echo "   *.${match}";;
+      esac
+    fi
+  done
+fi
+
+# Get version sorted *.domains filenames (without dir path)
+lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)")
+
+# Query blocklists for occurences of domain
+mapfile -t results <<< "$(scanList "${domainQuery}" "${lists[*]}" "${exact}")"
+
+# Handle notices
+if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then
+  echo -e "  ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the block lists"
+  exit 0
+elif [[ -z "${results[*]}" ]]; then
+  # Result found in WL/BL/Wildcards
+  exit 0
+elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then
+  echo -e "  ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC}
+    This can be overridden using the -all option"
+  exit 0
+fi
+
+# Remove unwanted content from non-exact $results
+if [[ -z "${exact}" ]]; then
+  # Delete lines starting with #
+  # Remove comments after domain
+  # Remove hosts format IP address
+  mapfile -t results <<< "$(IFS=$'\n'; sed \
+    -e "/:#/d" \
+    -e "s/[ \\t]#.*//g" \
+    -e "s/:.*[ \\t]/:/g" \
+  <<< "${results[*]}")"
+   # Exit if result was in a comment
+  [[ -z "${results[*]}" ]] && exit 0
+fi
+
+# Get adlist file content as array
+if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
+  for adlistUrl in $(< "adListsList"); do
+    if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then
+      adlists+=("${adlistUrl}")
+    fi
+  done
+fi
+
+# Print "Exact matches for" title
+if [[ -n "${exact}" ]] && [[ -z "${blockpage}" ]]; then
+  plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es"
+  echo " ${matchType^}${plural} for ${COL_BOLD}${domainQuery}${COL_NC} found in:"
+fi
+
+for result in "${results[@]}"; do
+  fileName="${result/:*/}"
+
+  # Determine *.domains URL using filename's number
+  if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
+    fileNum="${fileName/list./}"; fileNum="${fileNum%%.*}"
+    fileName="${adlists[$fileNum]}"
+
+    # Discrepency occurs when adlists has been modified, but Gravity has not been run
+    if [[ -z "${fileName}" ]]; then
+      fileName="${COL_LIGHT_RED}(no associated adlists URL found)${COL_NC}"
+    fi
+  fi
+
+  if [[ -n "${blockpage}" ]]; then
+    echo "${fileNum} ${fileName}"
+  elif [[ -n "${exact}" ]]; then
+    echo "   ${fileName}"
+  else
+    if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then
+      count=""
+      echo " ${matchType^} found in ${COL_BOLD}${fileName}${COL_NC}:"
+      fileName_prev="${fileName}"
+    fi
+    : $((count++))
+
+    # Print matching domain if $max_count has not been reached
+    [[ -z "${all}" ]] && max_count="50"
+    if [[ -z "${all}" ]] && [[ "${count}" -ge "${max_count}" ]]; then
+      [[ "${count}" -gt "${max_count}" ]] && continue
+      echo "   ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}"
+    else
+      echo "   ${result#*:}"
+    fi
+  fi
+done
+
+exit 0

advanced/Scripts/webpage.sh

@@ -204,10 +204,6 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423
 
 		add_dnsmasq_setting "interface" "${PIHOLE_INTERFACE}"
 	fi
-	if [[ "${CONDITIONAL_FORWARDING}" == true ]]; then
-		add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_DOMAIN}/${CONDITIONAL_FORWARDING_IP}"
-		add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_REVERSE}/${CONDITIONAL_FORWARDING_IP}"
-	fi
 
 }
 
@@ -237,17 +233,6 @@ SetDNSServers() {
 	else
 		change_setting "DNSSEC" "false"
 	fi
-	if [[ "${args[6]}" == "conditional_forwarding" ]]; then
-		change_setting "CONDITIONAL_FORWARDING" "true"
-		change_setting "CONDITIONAL_FORWARDING_IP" "${args[7]}"
-		change_setting "CONDITIONAL_FORWARDING_DOMAIN" "${args[8]}"
-		change_setting "CONDITIONAL_FORWARDING_REVERSE" "${args[9]}"
-	else
-		change_setting "CONDITIONAL_FORWARDING" "false"
-		delete_setting "CONDITIONAL_FORWARDING_IP"
-		delete_setting "CONDITIONAL_FORWARDING_DOMAIN"
-		delete_setting "CONDITIONAL_FORWARDING_REVERSE"
-	fi
 
 	ProcessDNSSettings
 
@@ -383,7 +368,9 @@ CustomizeAdLists() {
 	elif [[ "${args[2]}" == "disable" ]]; then
 		sed -i "\\@${args[3]}@s/^http/#http/g" "${list}"
 	elif [[ "${args[2]}" == "add" ]]; then
-		echo "${args[3]}" >> ${list}
+		if [[ $(grep -c "^${args[3]}$" "${list}") -eq 0 ]] ; then
+			echo "${args[3]}" >> ${list}
+		fi
 	elif [[ "${args[2]}" == "del" ]]; then
 	  var=$(echo "${args[3]}" | sed 's/\//\\\//g')
 	  sed -i "/${var}/Id" "${list}"