Fix merge conflicts

Signed-off-by: Adam Warner <me@adamwarner.co.uk>

advanced/Scripts/database_migration/gravity-db.sh

@@ -104,4 +104,10 @@ upgrade_gravityDB(){
 		sqlite3 "${database}" < "${scriptPath}/10_to_11.sql"
 		version=11
 	fi
+	if [[ "$version" == "11" ]]; then
+		# Rename group 0 from "Unassociated" to "Default"
+		echo -e "  ${INFO} Upgrading gravity database from version 11 to 12"
+		sqlite3 "${database}" < "${scriptPath}/11_to_12.sql"
+		version=12
+	fi
 }

advanced/Scripts/database_migration/gravity/11_to_12.sql

@@ -0,0 +1,19 @@
+.timeout 30000
+
+PRAGMA FOREIGN_KEYS=OFF;
+
+BEGIN TRANSACTION;
+
+UPDATE "group" SET name = 'Default' WHERE id = 0;
+UPDATE "group" SET description = 'The default group' WHERE id = 0;
+
+DROP TRIGGER IF EXISTS tr_group_zero;
+
+CREATE TRIGGER tr_group_zero AFTER DELETE ON "group"
+    BEGIN
+      INSERT OR IGNORE INTO "group" (id,enabled,name,description) VALUES (0,1,'Default','The default group');
+    END;
+
+UPDATE info SET value = 12 WHERE property = 'version';
+
+COMMIT;

advanced/Scripts/list.sh

@@ -22,6 +22,9 @@ web=false
 domList=()
 
 typeId=""
+comment=""
+declare -i domaincount
+domaincount=0
 
 colfile="/opt/pihole/COL_TABLE"
 source ${colfile}
@@ -97,10 +100,12 @@ ValidateDomain() {
     fi
 
     if [[ -n "${validDomain}" ]]; then
-        domList=("${domList[@]}" ${validDomain})
+        domList=("${domList[@]}" "${validDomain}")
     else
         echo -e "  ${CROSS} ${domain} is not a valid argument or domain name!"
     fi
+
+    domaincount=$((domaincount+1))
 }
 
 ProcessDomainList() {
@@ -151,7 +156,12 @@ AddDomain() {
     reload=true
     # Insert only the domain here. The enabled and date_added fields will be filled
     # with their default values (enabled = true, date_added = current timestamp)
-    sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});"
+    if [[ -z "${comment}" ]]; then
+        sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});"
+    else
+        # also add comment when variable has been set through the "--comment" option
+        sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');"
+    fi
 }
 
 RemoveDomain() {
@@ -224,8 +234,16 @@ NukeList() {
     sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};"
 }
 
-for var in "$@"; do
-    case "${var}" in
+GetComment() {
+    comment="$1"
+    if [[ "${comment}" =~ [^a-zA-Z0-9_\#:/\.,\ -] ]]; then
+      echo "  ${CROSS} Found invalid characters in domain comment!"
+      exit
+    fi
+}
+
+while (( "$#" )); do
+    case "${1}" in
         "-w" | "whitelist"   ) typeId=0;;
         "-b" | "blacklist"   ) typeId=1;;
         "--white-regex" | "white-regex" ) typeId=2;;
@@ -239,13 +257,15 @@ for var in "$@"; do
         "-l" | "--list"      ) Displaylist;;
         "--nuke"             ) NukeList;;
         "--web"              ) web=true;;
-        *                    ) ValidateDomain "${var}";;
+        "--comment"          ) GetComment "${2}"; shift;;
+        *                    ) ValidateDomain "${1}";;
     esac
+    shift
 done
 
 shift
 
-if [[ $# = 0 ]]; then
+if [[ ${domaincount} == 0 ]]; then
     helpFunc
 fi
 

advanced/Scripts/piholeDebug.sh

@@ -1107,22 +1107,19 @@ show_db_entries() {
 }
 
 show_groups() {
-    show_db_entries "Groups" "SELECT id,name,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,description FROM \"group\"" "4 50 7 19 19 50"
+    show_db_entries "Groups" "SELECT id,CASE enabled WHEN '0' THEN '   0' WHEN '1' THEN '      1' ELSE enabled END enabled,name,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,description FROM \"group\"" "4 7 50 19 19 50"
 }
 
 show_adlists() {
-    show_db_entries "Adlists" "SELECT id,address,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM adlist" "4 100 7 19 19 50"
-    show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 4"
+    show_db_entries "Adlists" "SELECT id,CASE enabled WHEN '0' THEN '   0' WHEN '1' THEN '      1' ELSE enabled END enabled,GROUP_CONCAT(adlist_by_group.group_id) group_ids,address,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM adlist LEFT JOIN adlist_by_group ON adlist.id = adlist_by_group.adlist_id GROUP BY id;" "4 7 12 100 19 19 50"
 }
 
 show_domainlist() {
-    show_db_entries "Domainlist (0/1 = exact white-/blacklist, 2/3 = regex white-/blacklist)" "SELECT id,type,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM domainlist" "4 4 100 7 19 19 50"
-    show_db_entries "Domainlist groups" "SELECT * FROM domainlist_by_group" "10 10"
+    show_db_entries "Domainlist (0/1 = exact white-/blacklist, 2/3 = regex white-/blacklist)" "SELECT id,CASE type WHEN '0' THEN '0   ' WHEN '1' THEN ' 1  ' WHEN '2' THEN '  2 ' WHEN '3' THEN '   3' ELSE type END type,CASE enabled WHEN '0' THEN '   0' WHEN '1' THEN '      1' ELSE enabled END enabled,GROUP_CONCAT(domainlist_by_group.group_id) group_ids,domain,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM domainlist LEFT JOIN domainlist_by_group ON domainlist.id = domainlist_by_group.domainlist_id GROUP BY id;" "4 4 7 12 100 19 19 50"
 }
 
 show_clients() {
-    show_db_entries "Clients" "SELECT id,ip,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM client" "4 100 19 19 50"
-    show_db_entries "Client groups" "SELECT * FROM client_by_group" "10 10"
+    show_db_entries "Clients" "SELECT id,GROUP_CONCAT(client_by_group.group_id) group_ids,ip,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM client LEFT JOIN client_by_group ON client.id = client_by_group.client_id GROUP BY id;" "4 12 100 19 19 50"
 }
 
 analyze_gravity_list() {

advanced/Scripts/version.sh

@@ -84,6 +84,21 @@ getRemoteVersion(){
     # Get the version from the remote origin
     local daemon="${1}"
     local version
+    local cachedVersions
+    local arrCache
+    cachedVersions="/etc/pihole/GitHubVersions"
+
+    #If the above file exists, then we can read from that. Prevents overuse of Github API
+    if [[ -f "$cachedVersions" ]]; then
+        IFS=' ' read -r -a arrCache < "$cachedVersions"
+        case $daemon in
+          "pi-hole"   )  echo "${arrCache[0]}";;
+          "AdminLTE"  )  echo "${arrCache[1]}";;
+          "FTL"       )  echo "${arrCache[2]}";;
+        esac
+
+        return 0
+    fi
 
     version=$(curl --silent --fail "https://api.github.com/repos/pi-hole/${daemon}/releases/latest" | \
         awk -F: '$1 ~/tag_name/ { print $2 }' | \
@@ -97,22 +112,48 @@ getRemoteVersion(){
     return 0
 }
 
+getLocalBranch(){
+    # Get the checked out branch of the local directory
+    local directory="${1}"
+    local branch
+
+     # Local FTL btranch is stored in /etc/pihole/ftlbranch
+    if [[ "$1" == "FTL" ]]; then
+        branch="$(pihole-FTL branch)"
+    else
+        cd "${directory}" 2> /dev/null || { echo "${DEFAULT}"; return 1; }
+        branch=$(git rev-parse --abbrev-ref HEAD || echo "$DEFAULT")
+    fi
+    if [[ ! "${branch}" =~ ^v ]]; then
+        if [[ "${branch}" == "master" ]]; then
+            echo ""
+        elif [[ "${branch}" == "HEAD" ]]; then
+            echo "in detached HEAD state at "
+        else
+            echo "${branch} "
+        fi
+    else
+        # Branch started in "v"
+        echo "release "
+    fi
+    return 0
+}
+
 versionOutput() {
     [[ "$1" == "pi-hole" ]] && GITDIR=$COREGITDIR
     [[ "$1" == "AdminLTE" ]] && GITDIR=$WEBGITDIR
     [[ "$1" == "FTL" ]] && GITDIR="FTL"
 
-    [[ "$2" == "-c" ]] || [[ "$2" == "--current" ]] || [[ -z "$2" ]] && current=$(getLocalVersion $GITDIR)
+    [[ "$2" == "-c" ]] || [[ "$2" == "--current" ]] || [[ -z "$2" ]] && current=$(getLocalVersion $GITDIR) && branch=$(getLocalBranch $GITDIR)
     [[ "$2" == "-l" ]] || [[ "$2" == "--latest" ]] || [[ -z "$2" ]] && latest=$(getRemoteVersion "$1")
     if [[ "$2" == "-h" ]] || [[ "$2" == "--hash" ]]; then
-        [[ "$3" == "-c" ]] || [[ "$3" == "--current" ]] || [[ -z "$3" ]] && curHash=$(getLocalHash "$GITDIR")
+        [[ "$3" == "-c" ]] || [[ "$3" == "--current" ]] || [[ -z "$3" ]] && curHash=$(getLocalHash "$GITDIR") && branch=$(getLocalBranch $GITDIR)
         [[ "$3" == "-l" ]] || [[ "$3" == "--latest" ]] || [[ -z "$3" ]] && latHash=$(getRemoteHash "$1" "$(cd "$GITDIR" 2> /dev/null && git rev-parse --abbrev-ref HEAD)")
     fi
-
     if [[ -n "$current" ]] && [[ -n "$latest" ]]; then
-        output="${1^} version is $current (Latest: $latest)"
+        output="${1^} version is $branch$current (Latest: $latest)"
     elif [[ -n "$current" ]] && [[ -z "$latest" ]]; then
-        output="Current ${1^} version is $current"
+        output="Current ${1^} version is $branch$current."
     elif [[ -z "$current" ]] && [[ -n "$latest" ]]; then
         output="Latest ${1^} version is $latest"
     elif [[ "$curHash" == "N/A" ]] || [[ "$latHash" == "N/A" ]]; then

advanced/Scripts/webpage.sh

@@ -36,7 +36,6 @@ Options:
   -c, celsius         Set Celsius as preferred temperature unit
   -f, fahrenheit      Set Fahrenheit as preferred temperature unit
   -k, kelvin          Set Kelvin as preferred temperature unit
-  -r, hostrecord      Add a name to the DNS associated to an IPv4/IPv6 address
   -e, email           Set an administrative contact address for the Block Page
   -h, --help          Show this help dialog
   -i, interface       Specify dnsmasq's interface listening behavior
@@ -478,32 +477,6 @@ RemoveDHCPStaticAddress() {
     sed -i "/dhcp-host=${mac}.*/d" "${dhcpstaticconfig}"
 }
 
-SetHostRecord() {
-    if [[ "${1}" == "-h" ]] || [[ "${1}" == "--help" ]]; then
-        echo "Usage: pihole -a hostrecord <domain> [IPv4-address],[IPv6-address]
-Example: 'pihole -a hostrecord home.domain.com 192.168.1.1,2001:db8:a0b:12f0::1'
-Add a name to the DNS associated to an IPv4/IPv6 address
-
-Options:
-  \"\"                  Empty: Remove host record
-  -h, --help          Show this help dialog"
-        exit 0
-    fi
-
-    if [[ -n "${args[3]}" ]]; then
-        change_setting "HOSTRECORD" "${args[2]},${args[3]}"
-        echo -e "  ${TICK} Setting host record for ${args[2]} to ${args[3]}"
-    else
-        change_setting "HOSTRECORD" ""
-        echo -e "  ${TICK} Removing host record"
-    fi
-
-    ProcessDNSSettings
-
-    # Restart dnsmasq to load new configuration
-    RestartDNS
-}
-
 SetAdminEmail() {
     if [[ "${1}" == "-h" ]] || [[ "${1}" == "--help" ]]; then
         echo "Usage: pihole -a email <address>
@@ -519,7 +492,10 @@ Options:
     if [[ -n "${args[2]}" ]]; then
 
         # Sanitize email address in case of security issues
-        if [[ ! "${args[2]}" =~ ^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,4}$  ]]; then
+        # Regex from https://stackoverflow.com/a/2138832/4065967
+        local regex
+        regex="^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\$"
+        if [[ ! "${args[2]}" =~ ${regex} ]]; then
             echo -e "  ${CROSS} Invalid email address"
             exit 0
         fi
@@ -667,7 +643,6 @@ main() {
         "resolve"             ) ResolutionSettings;;
         "addstaticdhcp"       ) AddDHCPStaticAddress;;
         "removestaticdhcp"    ) RemoveDHCPStaticAddress;;
-        "-r" | "hostrecord"   ) SetHostRecord "$3";;
         "-e" | "email"        ) SetAdminEmail "$3";;
         "-i" | "interface"    ) SetListeningMode "$@";;
         "-t" | "teleporter"   ) Teleporter;;

advanced/Templates/gravity.db.sql

@@ -10,7 +10,7 @@ CREATE TABLE "group"
 	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
 	description TEXT
 );
-INSERT INTO "group" (id,enabled,name) VALUES (0,1,'Unassociated');
+INSERT INTO "group" (id,enabled,name,description) VALUES (0,1,'Default','The default group');
 
 CREATE TABLE domainlist
 (
@@ -52,7 +52,7 @@ CREATE TABLE info
 	value TEXT NOT NULL
 );
 
-INSERT INTO "info" VALUES('version','11');
+INSERT INTO "info" VALUES('version','12');
 
 CREATE TABLE domain_audit
 (
@@ -167,7 +167,7 @@ CREATE TRIGGER tr_group_update AFTER UPDATE ON "group"
 
 CREATE TRIGGER tr_group_zero AFTER DELETE ON "group"
     BEGIN
-      INSERT OR IGNORE INTO "group" (id,enabled,name) VALUES (0,1,'Unassociated');
+      INSERT OR IGNORE INTO "group" (id,enabled,name) VALUES (0,1,'Default');
     END;
 
 CREATE TRIGGER tr_domainlist_delete AFTER DELETE ON domainlist

advanced/bash-completion/pihole

@@ -15,7 +15,7 @@ _pihole() {
 			COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) )
 		;;
 		"admin")
-			opts_admin="celsius email fahrenheit hostrecord interface kelvin password privacylevel"
+			opts_admin="celsius email fahrenheit interface kelvin password privacylevel"
 			COMPREPLY=( $(compgen -W "${opts_admin}" -- ${cur}) )
 		;;
 		"checkout")

advanced/index.php

@@ -6,8 +6,8 @@
 *  This file is copyright under the latest version of the EUPL.
 *  Please see LICENSE file for your rights under this license. */
 
-// Sanitize HTTP_HOST output
-$serverName = htmlspecialchars($_SERVER["HTTP_HOST"]);
+// Sanitize SERVER_NAME output
+$serverName = htmlspecialchars($_SERVER["SERVER_NAME"]);
 // Remove external ipv6 brackets if any
 $serverName = preg_replace('/^\[(.*)\]$/', '${1}', $serverName);
 
@@ -50,16 +50,24 @@ function setHeader($type = "x") {
 }
 
 // Determine block page type
-if ($serverName === "pi.hole") {
+if ($serverName === "pi.hole"
+    || (!empty($_SERVER["VIRTUAL_HOST"]) && $serverName === $_SERVER["VIRTUAL_HOST"])) {
     // Redirect to Web Interface
     exit(header("Location: /admin"));
 } elseif (filter_var($serverName, FILTER_VALIDATE_IP) || in_array($serverName, $authorizedHosts)) {
     // Set Splash Page output
     $splashPage = "
-    <html><head>
+    <html>
+      <head>
         $viewPort
-        <link rel='stylesheet' href='/pihole/blockingpage.css' type='text/css'/>
-    </head><body id='splashpage'><img src='/admin/img/logo.svg'/><br/>Pi-<b>hole</b>: Your black hole for Internet advertisements<br><a href='/admin'>Did you mean to go to the admin panel?</a></body></html>
+        <link rel='stylesheet' href='pihole/blockingpage.css' type='text/css'/>
+      </head>
+      <body id='splashpage'>
+        <img src='admin/img/logo.svg'/><br/>
+        Pi-<b>hole</b>: Your black hole for Internet advertisements<br/>
+        <a href='/admin'>Did you mean to go to the admin panel?</a>
+      </body>
+    </html>
     ";
 
     // Set splash/landing page based off presence of $landPage
@@ -131,7 +139,12 @@ ini_set("default_socket_timeout", 3);
 function queryAds($serverName) {
     // Determine the time it takes while querying adlists
     $preQueryTime = microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"];
-    $queryAds = file("http://127.0.0.1/admin/scripts/pi-hole/php/queryads.php?domain=$serverName&bp", FILE_IGNORE_NEW_LINES);
+    $queryAdsURL = sprintf(
+        "http://127.0.0.1:%s/admin/scripts/pi-hole/php/queryads.php?domain=%s&bp",
+        $_SERVER["SERVER_PORT"],
+        $serverName
+    );
+    $queryAds = file($queryAdsURL, FILE_IGNORE_NEW_LINES);
     $queryAds = array_values(array_filter(preg_replace("/data:\s+/", "", $queryAds)));
     $queryTime = sprintf("%.0f", (microtime(true)-$_SERVER["REQUEST_TIME_FLOAT"]) - $preQueryTime);
 
@@ -226,10 +239,10 @@ setHeader();
   <?=$viewPort ?>
   <meta name="robots" content="noindex,nofollow"/>
   <meta http-equiv="x-dns-prefetch-control" content="off">
-  <link rel="shortcut icon" href="//pi.hole/admin/img/favicon.png" type="image/x-icon"/>
-  <link rel="stylesheet" href="//pi.hole/pihole/blockingpage.css" type="text/css"/>
+  <link rel="shortcut icon" href="admin/img/favicon.png" type="image/x-icon"/>
+  <link rel="stylesheet" href="pihole/blockingpage.css" type="text/css"/>
   <title>● <?=$serverName ?></title>
-  <script src="//pi.hole/admin/scripts/vendor/jquery.min.js"></script>
+  <script src="admin/scripts/vendor/jquery.min.js"></script>
   <script>
     window.onload = function () {
       <?php

automated install/basic-install.sh

@@ -1214,11 +1214,10 @@ chooseBlocklists() {
         mv "${adlistFile}" "${adlistFile}.old"
     fi
     # Let user select (or not) blocklists via a checklist
-    cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 6)
+    cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 5)
     # In an array, show the options available (all off by default):
     options=(StevenBlack "StevenBlack's Unified Hosts List" on
         MalwareDom "MalwareDomains" on
-        Cameleon "Cameleon" on
         DisconTrack "Disconnect.me Tracking" on
         DisconAd "Disconnect.me Ads" on)
 
@@ -1239,7 +1238,6 @@ appendToListsFile() {
     case $1 in
         StevenBlack  )  echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";;
         MalwareDom   )  echo "https://mirror1.malwaredomains.com/files/justdomains" >> "${adlistFile}";;
-        Cameleon     )  echo "https://sysctl.org/cameleon/hosts" >> "${adlistFile}";;
         DisconTrack  )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" >> "${adlistFile}";;
         DisconAd     )  echo "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt" >> "${adlistFile}";;
     esac
@@ -1255,10 +1253,8 @@ installDefaultBlocklists() {
     fi
     appendToListsFile StevenBlack
     appendToListsFile MalwareDom
-    appendToListsFile Cameleon
     appendToListsFile DisconTrack
     appendToListsFile DisconAd
-    appendToListsFile HostsFile
 }
 
 # Check if /etc/dnsmasq.conf is from pi-hole.  If so replace with an original and install new in .d directory
@@ -1823,45 +1819,6 @@ create_pihole_user() {
     fi
 }
 
-# Allow HTTP and DNS traffic
-configureFirewall() {
-    printf "\\n"
-    # If a firewall is running,
-    if firewall-cmd --state &> /dev/null; then
-        # ask if the user wants to install Pi-hole's default firewall rules
-        whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" "${r}" "${c}" || \
-        { printf "  %b Not installing firewall rulesets.\\n" "${INFO}"; return 0; }
-        printf "  %b Configuring FirewallD for httpd and pihole-FTL\\n" "${TICK}"
-        # Allow HTTP and DNS traffic
-        firewall-cmd --permanent --add-service=http --add-service=dns
-        # Reload the firewall to apply these changes
-        firewall-cmd --reload
-        return 0
-        # Check for proper kernel modules to prevent failure
-    elif modinfo ip_tables &> /dev/null && is_command iptables ; then
-    # If chain Policy is not ACCEPT or last Rule is not ACCEPT
-    # then check and insert our Rules above the DROP/REJECT Rule.
-        if iptables -S INPUT | head -n1 | grep -qv '^-P.*ACCEPT$' || iptables -S INPUT | tail -n1 | grep -qv '^-\(A\|P\).*ACCEPT$'; then
-            whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" "${r}" "${c}" || \
-            { printf "  %b Not installing firewall rulesets.\\n" "${INFO}"; return 0; }
-            printf "  %b Installing new IPTables firewall rulesets\\n" "${TICK}"
-            # Check chain first, otherwise a new rule will duplicate old ones
-            iptables -C INPUT -p tcp -m tcp --dport 80 -j ACCEPT &> /dev/null || iptables -I INPUT 1 -p tcp -m tcp --dport 80 -j ACCEPT
-            iptables -C INPUT -p tcp -m tcp --dport 53 -j ACCEPT &> /dev/null || iptables -I INPUT 1 -p tcp -m tcp --dport 53 -j ACCEPT
-            iptables -C INPUT -p udp -m udp --dport 53 -j ACCEPT &> /dev/null || iptables -I INPUT 1 -p udp -m udp --dport 53 -j ACCEPT
-            iptables -C INPUT -p tcp -m tcp --dport 4711:4720 -i lo -j ACCEPT &> /dev/null || iptables -I INPUT 1 -p tcp -m tcp --dport 4711:4720 -i lo -j ACCEPT
-            return 0
-        fi
-    # Otherwise,
-    else
-        # no firewall is running
-        printf "  %b No active firewall detected.. skipping firewall configuration\\n" "${INFO}"
-        # so just exit
-        return 0
-    fi
-    printf "  %b Skipping firewall configuration\\n" "${INFO}"
-}
-
 #
 finalExports() {
     # If the Web interface is not set to be installed,
@@ -2010,10 +1967,6 @@ installPihole() {
     # Check if dnsmasq is present. If so, disable it and back up any possible
     # config file
     disable_dnsmasq
-    # Configure the firewall
-    if [[ "${useUpdateVars}" == false ]]; then
-        configureFirewall
-    fi
 
     # install a man page entry for pihole
     install_manpage

gravity.sh

@@ -376,7 +376,7 @@ gravity_DownloadBlocklists() {
     echo -e "  ${INFO} Target: ${url}"
     local regex
     # Check for characters NOT allowed in URLs
-    regex="[^a-zA-Z0-9:/?&%=~._-]"
+    regex="[^a-zA-Z0-9:/?&%=~._()-]"
     if [[ "${url}" =~ ${regex} ]]; then
         echo -e "  ${CROSS} Invalid Target"
     else
@@ -573,12 +573,14 @@ gravity_ParseFileIntoDomains() {
     # It also helps with debugging so each stage of the script can be researched more in depth
     # 1) Remove carriage returns
     # 2) Convert all characters to lowercase
-    # 3) Remove lines containing "#" or "/"
-    # 4) Remove leading tabs, spaces, etc.
-    # 5) Delete lines not matching domain names
+    # 3) Remove comments (text starting with "#", include possible spaces before the hash sign)
+    # 4) Remove lines containing "/"
+    # 5) Remove leading tabs, spaces, etc.
+    # 6) Delete lines not matching domain names
     < "${source}" tr -d '\r' | \
     tr '[:upper:]' '[:lower:]' | \
-    sed -r '/(\/|#).*$/d' | \
+    sed 's/\s*#.*//g' | \
+    sed -r '/(\/).*$/d' | \
     sed -r 's/^.*\s+//g' | \
     sed -r '/([^\.]+\.)+[^\.]{2,}/!d' >  "${destination}"
     chmod 644 "${destination}"
@@ -638,7 +640,7 @@ gravity_Table_Count() {
   if [[ "${table}" == "vw_gravity" ]]; then
     local unique
     unique="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(DISTINCT domain) FROM ${table};")"
-    echo -e "  ${INFO} Number of ${str}: ${num} (${unique} unique domains)"
+    echo -e "  ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})"
     sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});"
   else
     echo -e "  ${INFO} Number of ${str}: ${num}"

manpages/pihole.8

@@ -1,4 +1,4 @@
-.TH "Pi-hole" "8" "Pi-hole" "Pi-hole" "May 2018"
+.TH "Pi-hole" "8" "Pi-hole" "Pi-hole" "April 2020"
 .SH "NAME"
 
 Pi-hole : A black-hole for internet advertisements
@@ -11,8 +11,6 @@ Pi-hole : A black-hole for internet advertisements
 .br
 \fBpihole -a\fR (\fB-c|-f|-k\fR)
 .br
-\fBpihole -a\fR [\fB-r\fR hostrecord]
-.br
 \fBpihole -a -e\fR email
 .br
 \fBpihole -a -i\fR interface
@@ -43,7 +41,7 @@ pihole -g\fR
 .br
 pihole status
 .br
-pihole restartdns\fR
+pihole restartdns\fR [options]
 .br
 \fBpihole\fR (\fBenable\fR|\fBdisable\fR [time])
 .br
@@ -134,9 +132,6 @@ Available commands and options:
       -f, fahrenheit    Set Fahrenheit as preferred temperature unit
 .br
       -k, kelvin        Set Kelvin as preferred temperature unit
-.br
-      -r, hostrecord    Add a name to the DNS associated to an
-                        IPv4/IPv6 address
 .br
       -e, email         Set an administrative contact address for the
                         Block Page
@@ -260,9 +255,16 @@ Available commands and options:
       #m                Disable Pi-hole functionality for # minute(s)
 .br
 
-\fBrestartdns\fR
+\fBrestartdns\fR [options]
 .br
-    Restart Pi-hole subsystems
+    Full restart Pi-hole subsystems
+.br
+
+    (restart options):
+.br
+      reload            Updates the lists and flushes DNS cache
+.br
+      reload-lists      Updates the lists WITHOUT flushing the DNS cache
 .br
 
 \fBcheckout\fR [repo] [branch]

pihole

@@ -413,7 +413,9 @@ Options:
   enable              Enable Pi-hole subsystems
   disable             Disable Pi-hole subsystems
                         Add '-h' for more info on disable usage
-  restartdns          Restart Pi-hole subsystems
+  restartdns          Full restart Pi-hole subsystems
+                        Add 'reload' to update the lists and flush the cache without restarting the DNS server
+                        Add 'reload-lists' to only update the lists WITHOUT flushing the cache or restarting the DNS server
   checkout            Switch Pi-hole subsystems to a different Github branch
                         Add '-h' for more info on checkout usage
   arpflush            Flush information stored in Pi-hole's network tables";

test/test_automated_install.py

@@ -92,213 +92,6 @@ def test_setupVars_saved_to_file(Pihole):
         assert "{}={}".format(k, v) in output
 
 
-def test_pihole_user_group_creation(Pihole):
-    '''
-    check user creation works if user or group already exist
-    '''
-    sudo_cmd = 'su --shell /bin/bash --command "{0}" -p root'
-    # normal situation where neither user or group exist
-    user_create = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    create_pihole_user
-    ''')
-    expected_stdout = tick_box + ' Creating user \'pihole\''
-    assert expected_stdout in user_create.stdout
-    # situation where both user and group already exist
-    user_create = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    create_pihole_user
-    ''')
-    expected_stdout = tick_box + ' Checking for user \'pihole\''
-    assert expected_stdout in user_create.stdout
-    # situation where only group and no user exists
-    Pihole.run(sudo_cmd.format('userdel -r pihole'))
-    user_create = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    create_pihole_user
-    ''')
-    expected_stdout = tick_box + ' Creating user \'pihole\''
-    assert expected_stdout in user_create.stdout
-    # situation where only user and no group exists
-    Pihole.run(sudo_cmd.format('userdel -r pihole'))
-    Pihole.run(sudo_cmd.format('groupdel pihole'))
-    Pihole.run(sudo_cmd.format('groupadd pihole_dummy'))
-    useradd_dummy = (
-        'useradd -r --no-user-group -g pihole_dummy ' +
-        '-s /usr/sbin/nologin pihole')
-    Pihole.run(sudo_cmd.format(useradd_dummy))
-    user_create = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    create_pihole_user
-    ''')
-    expected_stdout = tick_box + ' Creating group \'pihole\''
-    assert expected_stdout in user_create.stdout
-    expected_stdout = tick_box + ' Adding user \'pihole\' to group \'pihole\''
-    assert expected_stdout in user_create.stdout
-
-
-def test_configureFirewall_firewalld_running_no_errors(Pihole):
-    '''
-    confirms firewalld rules are applied when firewallD is running
-    '''
-    # firewallD returns 'running' as status
-    mock_command('firewall-cmd', {'*': ('running', 0)}, Pihole)
-    # Whiptail dialog returns Ok for user prompt
-    mock_command('whiptail', {'*': ('', 0)}, Pihole)
-    configureFirewall = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    configureFirewall
-    ''')
-    expected_stdout = 'Configuring FirewallD for httpd and pihole-FTL'
-    assert expected_stdout in configureFirewall.stdout
-    firewall_calls = Pihole.run('cat /var/log/firewall-cmd').stdout
-    assert 'firewall-cmd --state' in firewall_calls
-    assert ('firewall-cmd '
-            '--permanent '
-            '--add-service=http '
-            '--add-service=dns') in firewall_calls
-    assert 'firewall-cmd --reload' in firewall_calls
-
-
-def test_configureFirewall_firewalld_disabled_no_errors(Pihole):
-    '''
-    confirms firewalld rules are not applied when firewallD is not running
-    '''
-    # firewallD returns non-running status
-    mock_command('firewall-cmd', {'*': ('not running', '1')}, Pihole)
-    configureFirewall = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    configureFirewall
-    ''')
-    expected_stdout = ('No active firewall detected.. '
-                       'skipping firewall configuration')
-    assert expected_stdout in configureFirewall.stdout
-
-
-def test_configureFirewall_firewalld_enabled_declined_no_errors(Pihole):
-    '''
-    confirms firewalld rules are not applied when firewallD is running, user
-    declines ruleset
-    '''
-    # firewallD returns running status
-    mock_command('firewall-cmd', {'*': ('running', 0)}, Pihole)
-    # Whiptail dialog returns Cancel for user prompt
-    mock_command('whiptail', {'*': ('', 1)}, Pihole)
-    configureFirewall = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    configureFirewall
-    ''')
-    expected_stdout = 'Not installing firewall rulesets.'
-    assert expected_stdout in configureFirewall.stdout
-
-
-def test_configureFirewall_no_firewall(Pihole):
-    ''' confirms firewall skipped no daemon is running '''
-    configureFirewall = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    configureFirewall
-    ''')
-    expected_stdout = 'No active firewall detected'
-    assert expected_stdout in configureFirewall.stdout
-
-
-def test_configureFirewall_IPTables_enabled_declined_no_errors(Pihole):
-    '''
-    confirms IPTables rules are not applied when IPTables is running, user
-    declines ruleset
-    '''
-    # iptables command exists
-    mock_command('iptables', {'*': ('', '0')}, Pihole)
-    # modinfo returns always true (ip_tables module check)
-    mock_command('modinfo', {'*': ('', '0')}, Pihole)
-    # Whiptail dialog returns Cancel for user prompt
-    mock_command('whiptail', {'*': ('', '1')}, Pihole)
-    configureFirewall = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    configureFirewall
-    ''')
-    expected_stdout = 'Not installing firewall rulesets.'
-    assert expected_stdout in configureFirewall.stdout
-
-
-def test_configureFirewall_IPTables_enabled_rules_exist_no_errors(Pihole):
-    '''
-    confirms IPTables rules are not applied when IPTables is running and rules
-    exist
-    '''
-    # iptables command exists and returns 0 on calls
-    # (should return 0 on iptables -C)
-    mock_command('iptables', {'-S': ('-P INPUT DENY', '0')}, Pihole)
-    # modinfo returns always true (ip_tables module check)
-    mock_command('modinfo', {'*': ('', '0')}, Pihole)
-    # Whiptail dialog returns Cancel for user prompt
-    mock_command('whiptail', {'*': ('', '0')}, Pihole)
-    configureFirewall = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    configureFirewall
-    ''')
-    expected_stdout = 'Installing new IPTables firewall rulesets'
-    assert expected_stdout in configureFirewall.stdout
-    firewall_calls = Pihole.run('cat /var/log/iptables').stdout
-    # General call type occurrences
-    assert len(re.findall(r'iptables -S', firewall_calls)) == 1
-    assert len(re.findall(r'iptables -C', firewall_calls)) == 4
-    assert len(re.findall(r'iptables -I', firewall_calls)) == 0
-
-    # Specific port call occurrences
-    assert len(re.findall(r'tcp --dport 80', firewall_calls)) == 1
-    assert len(re.findall(r'tcp --dport 53', firewall_calls)) == 1
-    assert len(re.findall(r'udp --dport 53', firewall_calls)) == 1
-    assert len(re.findall(r'tcp --dport 4711:4720', firewall_calls)) == 1
-
-
-def test_configureFirewall_IPTables_enabled_not_exist_no_errors(Pihole):
-    '''
-    confirms IPTables rules are applied when IPTables is running and rules do
-    not exist
-    '''
-    # iptables command and returns 0 on calls (should return 1 on iptables -C)
-    mock_command(
-        'iptables',
-        {
-            '-S': (
-                '-P INPUT DENY',
-                '0'
-            ),
-            '-C': (
-                '',
-                1
-            ),
-            '-I': (
-                '',
-                0
-            )
-        },
-        Pihole
-    )
-    # modinfo returns always true (ip_tables module check)
-    mock_command('modinfo', {'*': ('', '0')}, Pihole)
-    # Whiptail dialog returns Cancel for user prompt
-    mock_command('whiptail', {'*': ('', '0')}, Pihole)
-    configureFirewall = Pihole.run('''
-    source /opt/pihole/basic-install.sh
-    configureFirewall
-    ''')
-    expected_stdout = 'Installing new IPTables firewall rulesets'
-    assert expected_stdout in configureFirewall.stdout
-    firewall_calls = Pihole.run('cat /var/log/iptables').stdout
-    # General call type occurrences
-    assert len(re.findall(r'iptables -S', firewall_calls)) == 1
-    assert len(re.findall(r'iptables -C', firewall_calls)) == 4
-    assert len(re.findall(r'iptables -I', firewall_calls)) == 4
-
-    # Specific port call occurrences
-    assert len(re.findall(r'tcp --dport 80', firewall_calls)) == 2
-    assert len(re.findall(r'tcp --dport 53', firewall_calls)) == 2
-    assert len(re.findall(r'udp --dport 53', firewall_calls)) == 2
-    assert len(re.findall(r'tcp --dport 4711:4720', firewall_calls)) == 2
-
-
 def test_selinux_not_detected(Pihole):
     '''
     confirms installer continues when SELinux configuration file does not exist