diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 40fa9655..7eba43c0 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -22,6 +22,7 @@ upgrade_gravityDB(){ if [[ "$version" == "1" ]]; then # This migration script upgrades the gravity.db file by # adding the domain_audit table + echo -e " ${INFO} Upgrading gravity database from version 1 to 2" sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" version=2 @@ -36,6 +37,7 @@ upgrade_gravityDB(){ # This migration script upgrades the gravity.db file by # renaming the regex table to regex_blacklist, and # creating a new regex_whitelist table + corresponding linking table and views + echo -e " ${INFO} Upgrading gravity database from version 2 to 3" sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/2_to_3.sql" version=3 fi diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh old mode 100755 new mode 100644 index f40adfbf..1e1b159c --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -183,6 +183,20 @@ lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)") # Query blocklists for occurences of domain mapfile -t results <<< "$(scanList "${domainQuery}" "${lists[*]}" "${exact}")" +# Remove unwanted content from $results +# Each line in $results is formatted as such: [fileName]:[line] +# 1. Delete lines starting with # +# 2. Remove comments after domain +# 3. Remove hosts format IP address +# 4. Remove any lines that no longer contain the queried domain name (in case the matched domain name was in a comment) +esc_domain="${domainQuery//./\\.}" +mapfile -t results <<< "$(IFS=$'\n'; sed \ + -e "/:#/d" \ + -e "s/[ \\t]#.*//g" \ + -e "s/:.*[ \\t]/:/g" \ + -e "/${esc_domain}/!d" \ + <<< "${results[*]}")" + # Handle notices if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the block lists" @@ -196,20 +210,6 @@ elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then exit 0 fi -# Remove unwanted content from non-exact $results -if [[ -z "${exact}" ]]; then - # Delete lines starting with # - # Remove comments after domain - # Remove hosts format IP address - mapfile -t results <<< "$(IFS=$'\n'; sed \ - -e "/:#/d" \ - -e "s/[ \\t]#.*//g" \ - -e "s/:.*[ \\t]/:/g" \ - <<< "${results[*]}")" - # Exit if result was in a comment - [[ -z "${results[*]}" ]] && exit 0 -fi - # Get adlist file content as array if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then # Retrieve source URLs from gravity database diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 39299960..411cc1f6 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -16,6 +16,7 @@ readonly dhcpconfig="/etc/dnsmasq.d/02-pihole-dhcp.conf" readonly FTLconf="/etc/pihole/pihole-FTL.conf" # 03 -> wildcards readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf" +readonly PI_HOLE_BIN_DIR="/usr/local/bin" readonly gravityDBfile="/etc/pihole/gravity.db" @@ -212,6 +213,11 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_DOMAIN}/${CONDITIONAL_FORWARDING_IP}" add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_REVERSE}/${CONDITIONAL_FORWARDING_IP}" fi + + # Prevent Firefox from automatically switching over to DNS-over-HTTPS + # This follows https://support.mozilla.org/en-US/kb/configuring-networks-disable-dns-over-https + # (sourced 7th September 2019) + add_dnsmasq_setting "server=/use-application-dns.net/" } SetDNSServers() { @@ -276,7 +282,7 @@ Reboot() { } RestartDNS() { - /usr/local/bin/pihole restartdns + "${PI_HOLE_BIN_DIR}"/pihole restartdns } SetQueryLogOptions() { diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 09d581f0..d0c744f4 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -89,10 +89,6 @@ CREATE TABLE info INSERT INTO info VALUES("version","1"); -CREATE VIEW vw_gravity AS SELECT domain - FROM gravity - WHERE domain NOT IN (SELECT domain from vw_whitelist); - CREATE VIEW vw_whitelist AS SELECT DISTINCT domain FROM whitelist LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id @@ -141,3 +137,6 @@ CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; END; +CREATE VIEW vw_gravity AS SELECT domain + FROM gravity + WHERE domain NOT IN (SELECT domain from vw_whitelist); diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 25c66ab7..cc78afbf 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -65,11 +65,11 @@ PI_HOLE_FILES=(chronometer list piholeDebug piholeLogFlush setupLCD update versi # This directory is where the Pi-hole scripts will be installed PI_HOLE_INSTALL_DIR="/opt/pihole" PI_HOLE_CONFIG_DIR="/etc/pihole" +PI_HOLE_BIN_DIR="/usr/local/bin" PI_HOLE_BLOCKPAGE_DIR="${webroot}/pihole" useUpdateVars=false adlistFile="/etc/pihole/adlists.list" -regexFile="/etc/pihole/regex.list" # Pi-hole needs an IP address; to begin, these variables are empty since we don't know what the IP is until # this script can run IPV4_ADDRESS="" @@ -1351,7 +1351,7 @@ installScripts() { install -o "${USER}" -Dm755 -t "${PI_HOLE_INSTALL_DIR}" ./advanced/Scripts/*.sh install -o "${USER}" -Dm755 -t "${PI_HOLE_INSTALL_DIR}" ./automated\ install/uninstall.sh install -o "${USER}" -Dm755 -t "${PI_HOLE_INSTALL_DIR}" ./advanced/Scripts/COL_TABLE - install -o "${USER}" -Dm755 -t /usr/local/bin/ pihole + install -o "${USER}" -Dm755 -t "${PI_HOLE_BIN_DIR}" pihole install -Dm644 ./advanced/bash-completion/pihole /etc/bash_completion.d/pihole printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" @@ -1384,11 +1384,6 @@ installConfigs() { return 1 fi fi - # Install an empty regex file - if [[ ! -f "${regexFile}" ]]; then - # Let PHP edit the regex file, if installed - install -o pihole -g "${LIGHTTPD_GROUP:-pihole}" -m 664 /dev/null "${regexFile}" - fi # If the user chose to install the dashboard, if [[ "${INSTALL_WEB_SERVER}" == true ]]; then # and if the Web server conf directory does not exist, @@ -1692,7 +1687,7 @@ installPiholeWeb() { # Otherwise, else # don't do anything - printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" + printf "%b %b %s\\n" "${OVER}" "${INFO}" "${str}" printf " No default index.lighttpd.html file found... not backing up\\n" fi @@ -1704,13 +1699,13 @@ installPiholeWeb() { # and copy in the pihole sudoers file install -m 0640 ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.sudo /etc/sudoers.d/pihole # Add lighttpd user (OS dependent) to sudoers file - echo "${LIGHTTPD_USER} ALL=NOPASSWD: /usr/local/bin/pihole" >> /etc/sudoers.d/pihole + echo "${LIGHTTPD_USER} ALL=NOPASSWD: ${PI_HOLE_BIN_DIR}/pihole" >> /etc/sudoers.d/pihole # If the Web server user is lighttpd, if [[ "$LIGHTTPD_USER" == "lighttpd" ]]; then # Allow executing pihole via sudo with Fedora - # Usually /usr/local/bin is not permitted as directory for sudoable programs - echo "Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin" >> /etc/sudoers.d/pihole + # Usually /usr/local/bin ${PI_HOLE_BIN_DIR} is not permitted as directory for sudoable programs + echo "Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:${PI_HOLE_BIN_DIR}" >> /etc/sudoers.d/pihole fi # Set the strict permissions on the file chmod 0440 /etc/sudoers.d/pihole @@ -2413,7 +2408,7 @@ FTLcheckUpdate() { return 3 fi - FTLlatesttag=$(grep 'Location' < "${FTLreleaseData}" | awk -F '/' '{print $NF}' | tr -d '\r\n') + FTLlatesttag=$(grep 'Location' <<< "${FTLreleaseData}" | awk -F '/' '{print $NF}' | tr -d '\r\n') if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then return 0 @@ -2699,7 +2694,7 @@ main() { if [[ "${INSTALL_TYPE}" == "Update" ]]; then printf "\\n" - /usr/local/bin/pihole version --current + "${PI_HOLE_BIN_DIR}"/pihole version --current fi } diff --git a/gravity.sh b/gravity.sh index 86bb6a2e..7a9e4f67 100755 --- a/gravity.sh +++ b/gravity.sh @@ -488,48 +488,7 @@ gravity_ParseFileIntoDomains() { # Determine how to parse individual source file formats if [[ "${firstLine,,}" =~ (adblock|ublock|^!) ]]; then # Compare $firstLine against lower case words found in Adblock lists - echo -ne " ${INFO} Format: Adblock" - - # Define symbols used as comments: [! - # "||.*^" includes the "Example 2" domains we can extract - # https://adblockplus.org/filter-cheatsheet - abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/" - - # Parse Adblock lists by extracting "Example 2" domains - # Logic: Ignore lines which do not include comments or domain name anchor - awk ''"${abpFilter}"' { - # Remove valid adblock type options - gsub(/\$?~?(important|third-party|popup|subdocument|websocket),?/, "", $0) - # Remove starting domain name anchor "||" and ending seperator "^" - gsub(/^(\|\|)|(\^)/, "", $0) - # Remove invalid characters (*/,=$) - if($0 ~ /[*\/,=\$]/) { $0="" } - # Remove lines which are only IPv4 addresses - if($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } - if($0) { print $0 } - }' "${source}" > "${destination}" - chmod 644 "${destination}" - - # Determine if there are Adblock exception rules - # https://adblockplus.org/filters - if grep -q "^@@||" "${source}" &> /dev/null; then - # Parse Adblock lists by extracting exception rules - # Logic: Ignore lines which do not include exception format "@@||example.com^" - awk -F "[|^]" '/^@@\|\|.*\^/ { - # Remove valid adblock type options - gsub(/\$?~?(third-party)/, "", $0) - # Remove invalid characters (*/,=$) - if($0 ~ /[*\/,=\$]/) { $0="" } - if($3) { print $3 } - }' "${source}" > "${destination}.exceptionsFile.tmp" - - # Remove exceptions - comm -23 "${destination}" <(sort "${destination}.exceptionsFile.tmp") > "${source}" - mv "${source}" "${destination}" - chmod 644 "${destination}" - fi - - echo -e "${OVER} ${TICK} Format: Adblock" + echo -e " ${CROSS} Format: Adblock (list type not supported)" elif grep -q "^address=/" "${source}" &> /dev/null; then # Parse Dnsmasq format lists echo -e " ${CROSS} Format: Dnsmasq (list type not supported)" diff --git a/pihole b/pihole index 1d9f0809..971595d5 100755 --- a/pihole +++ b/pihole @@ -11,10 +11,11 @@ readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" -# setupVars is not readonly here because in some functions (checkout), +# setupVars and PI_HOLE_BIN_DIR are not readonly here because in some funcitons (checkout), # it might get set again when the installer is sourced. This causes an # error due to modifying a readonly variable. setupVars="/etc/pihole/setupVars.conf" +PI_HOLE_BIN_DIR="/usr/local/bin" readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" source "${colfile}" @@ -101,20 +102,16 @@ versionFunc() { restartDNS() { local svcOption svc str output status - svcOption="${1:-}" + svcOption="${1:-restart}" - # Determine if we should reload or restart restart + # Determine if we should reload or restart if [[ "${svcOption}" =~ "reload" ]]; then - # Using SIGHUP will NOT re-read any *.conf files + # Reload has been requested + # Note: This will NOT re-read any *.conf files svc="killall -s SIGHUP ${resolver}" else - # Get PID of resolver to determine if it needs to start or restart - if pidof pihole-FTL &> /dev/null; then - svcOption="restart" - else - svcOption="start" - fi - svc="service ${resolver} ${svcOption}" + # A full restart has been requested + svc="service ${resolver} restart" fi # Print output to Terminal, but not to Web Admin @@ -159,7 +156,7 @@ Time: local str="Disabling blocking for ${tt} seconds" echo -e " ${INFO} ${str}..." local str="Blocking will be re-enabled in ${tt} seconds" - nohup bash -c "sleep ${tt}; pihole enable" /dev/null & + nohup bash -c "sleep ${tt}; ${PI_HOLE_BIN_DIR}/pihole enable" /dev/null & else local error=true fi @@ -170,7 +167,7 @@ Time: echo -e " ${INFO} ${str}..." local str="Blocking will be re-enabled in ${tt} minutes" tt=$((${tt}*60)) - nohup bash -c "sleep ${tt}; pihole enable" /dev/null & + nohup bash -c "sleep ${tt}; ${PI_HOLE_BIN_DIR}/pihole enable" /dev/null & else local error=true fi @@ -226,7 +223,7 @@ Options: sed -i 's/^QUERY_LOGGING=true/QUERY_LOGGING=false/' /etc/pihole/setupVars.conf if [[ "${2}" != "noflush" ]]; then # Flush logs - pihole -f + "${PI_HOLE_BIN_DIR}"/pihole -f fi echo -e " ${INFO} Disabling logging..." local str="Logging has been disabled!" @@ -279,7 +276,7 @@ statusFunc() { *) echo -e " ${INFO} Pi-hole blocking will be enabled";; esac # Enable blocking - pihole enable + "${PI_HOLE_BIN_DIR}"/pihole enable fi } diff --git a/test/test_automated_install.py b/test/test_automated_install.py index cce11857..e8a4dede 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -338,7 +338,7 @@ def test_installPiholeWeb_fresh_install_no_errors(Pihole): expected_stdout = tick_box + (' Creating directory for blocking page, ' 'and copying files') assert expected_stdout in installWeb.stdout - expected_stdout = cross_box + ' Backing up index.lighttpd.html' + expected_stdout = info_box + ' Backing up index.lighttpd.html' assert expected_stdout in installWeb.stdout expected_stdout = ('No default index.lighttpd.html file found... ' 'not backing up')