Pi-hole/pi-hole
Pi-hole/pi-hole
1
0
Fork 0
mirror of https://github.com/pi-hole/pi-hole.git synced 2024-11-15 10:43:55 +00:00
Code Issues Projects Releases Packages Wiki Activity

Optimised parsing of domains on IPv6 servers

Browse source 
* Remove WHITELIST_COMMAND
* Place IPv4/IPv6 availability test underneath setupVars.conf source
* Improved clarity on comments
* Define default lookupDomain on local line
* Use `getent hosts` instead of nslookup (faster)
* Make gravity_DNSLookup() function more readable
* Use bold on "Neutrino emissions detected"
* Swap conditionals around on adlists file handling
* Add comments to both gravity_Collapse() `awk`s
* Removed unnecessary "${str}" from gravity_Pull()
* Merge function variables into local line
* Place .phgbp suffice on mktemp, so patternbuffers can be cleaned up all at once in gravity_Cleanup()
* Removed success="false" from $httpCode case, placed empty success var in local
* Reordered $httpCode case numerically because I can
* Provide error if Dnsmasq format list is being parsed
* Remove IPv4 check when determining URL list (too slow on large lists)
* Check ${#sources[@]} to ensure we're checking the number of entries and not the character count
* Define empty plural in local line, removing unnecessary plural=;
* Optimised readability of gravity_Whitelist()
* Removed uninformative "Nothing to blacklist"/"No wildcards used" text
* Optimised parsing of domains into hosts format on IPv6 enabled servers
* Ensure /etc/hostname is non-zero
* Use `: >` instead of `rm` as consistent with the rest of the script
* Ensured that gravity_Cleanup() removes ${localList}.tmp
* Optimised readability of gravity_ParseUserDomains()
* Moved dnsRestart to ${var} case statement, renaming it to dnsRestartType for readability
* Set default $listType to ensure script passes "bash strict mode"
This commit is contained in:
WaLLy3K 2017-09-15 22:39:17 +10:00 committed by GitHub
parent ff5411a93a
commit c957124fad
1 changed files with 207 additions and 213 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

420
gravity.sh
View file

@ -16,7 +16,6 @@ source "${coltable}"
basename="pihole" basename="pihole"
PIHOLE_COMMAND="/usr/local/bin/${basename}" PIHOLE_COMMAND="/usr/local/bin/${basename}"
WHITELIST_COMMAND="${PIHOLE_COMMAND} -w"
piholeDir="/etc/${basename}" piholeDir="/etc/${basename}"
piholeRepo="/etc/.${basename}" piholeRepo="/etc/.${basename}"
@ -51,87 +50,99 @@ if [[ -f "${setupVars}" ]];then
# Remove CIDR mask from IPv4/6 addresses # Remove CIDR mask from IPv4/6 addresses
IPV4_ADDRESS="${IPV4_ADDRESS%/*}" IPV4_ADDRESS="${IPV4_ADDRESS%/*}"
IPV6_ADDRESS="${IPV6_ADDRESS%/*}" IPV6_ADDRESS="${IPV6_ADDRESS%/*}"
# Determine if IPv4/6 addresses exist
if [[ -z "${IPV4_ADDRESS}" ]] && [[ -z "${IPV6_ADDRESS}" ]]; then
echo -e " ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}"
exit 1
fi
else else
echo -e " ${COL_LIGHT_RED}Installation Failure: ${setupVars} does not exist! ${COL_NC} echo -e " ${COL_LIGHT_RED}Installation Failure: ${setupVars} does not exist! ${COL_NC}
Please run 'pihole -r', and choose the 'reconfigure' option to fix." Please run 'pihole -r', and choose the 'reconfigure' option to fix."
exit 1 exit 1
fi fi
# Warn users still using pihole.conf that it no longer has any effect # Determine if superseded pihole.conf exists
if [[ -r "${piholeDir}/pihole.conf" ]]; then if [[ -r "${piholeDir}/pihole.conf" ]]; then
echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}" echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}"
fi fi
# Determine if DNS resolution is available before proceeding with retrieving blocklists # Determine if DNS resolution is available before proceeding
gravity_DNSLookup() { gravity_DNSLookup() {
local lookupDomain plural local lookupDomain="pi.hole" plural=""
# Determine which domain should be resolved # Determine if $localList does not exist
# "pi.hole" is not always available (e.g: new install), but FTL will not log it if [[ ! -e "${localList}" ]]; then
if [[ -e "${localList}" ]]; then
lookupDomain="pi.hole"
else
lookupDomain="raw.githubusercontent.com" lookupDomain="raw.githubusercontent.com"
fi fi
# Determine if domain can be resolved # Determine if $lookupDomain is resolvable
if ! timeout 5 nslookup "${lookupDomain}" &> /dev/null; then if timeout 5 getent hosts "${lookupDomain}" &> /dev/null; then
if [[ -n "${secs}" ]]; then
echo -e "${OVER} ${CROSS} DNS resolution is still unavailable, cancelling"
exit 1
fi
# Determine error output message
if pidof dnsmasq &> /dev/null; then
echo -e " ${CROSS} DNS resolution is temporarily unavailable"
else
echo -e " ${CROSS} DNS service is not running"
"${PIHOLE_COMMAND}" restartdns
fi
# Give time for DNS server to be resolvable
secs="30"
while [[ "${secs}" -ne 0 ]]; do
plural=; [[ "${secs}" -ne 1 ]] && plural="s"
echo -ne "${OVER} ${INFO} Waiting $secs second${plural} before continuing..."
sleep 1
: $((secs--))
done
# Try again
gravity_DNSLookup
elif [[ -n "${secs}" ]]; then
# Print confirmation of resolvability if it had previously failed # Print confirmation of resolvability if it had previously failed
echo -e "${OVER} ${TICK} DNS resolution is now available\\n" if [[ -n "${secs:-}" ]]; then
echo -e "${OVER} ${TICK} DNS resolution is now available\\n"
fi
return 0
elif [[ -n "${secs:-}" ]]; then
echo -e "${OVER} ${CROSS} DNS resolution is still unavailable, cancelling"
exit 1
fi fi
# Determine error output message
if pidof dnsmasq &> /dev/null; then
echo -e " ${CROSS} DNS resolution is currently unavailable"
else
echo -e " ${CROSS} DNS service is not running"
"${PIHOLE_COMMAND}" restartdns
fi
# Give time for DNS server to be resolvable
secs="30"
while [[ "${secs}" -ne 0 ]]; do
[[ "${secs}" -ne 1 ]] && plural="s"
echo -ne "${OVER} ${INFO} Waiting $secs second${plural} before continuing..."
sleep 1
: $((secs--))
done
# Try again
gravity_DNSLookup
} }
# Retrieve blocklist URLs and parse domains from adlists.list # Retrieve blocklist URLs and parse domains from adlists.list
gravity_Collapse() { gravity_Collapse() {
echo -e " ${INFO} Neutrino emissions detected..." echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
# Handle "adlists.list" and "adlists.default" files # Determine if adlists file needs handling
if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then if [[ ! -f "${adListFile}" ]]; then
# Create "adlists.list" by copying "adlists.default" from internal core repo
cp "${adListRepoDefault}" "${adListFile}" 2> /dev/null || \
echo -e " ${CROSS} Unable to copy ${adListFile##*/} from ${piholeRepo}"
elif [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then
# Remove superceded $adListDefault file # Remove superceded $adListDefault file
rm "${adListDefault}" 2> /dev/null || \ rm "${adListDefault}" 2> /dev/null || \
echo -e " ${CROSS} Unable to remove ${adListDefault}" echo -e " ${CROSS} Unable to remove ${adListDefault}"
elif [[ ! -f "${adListFile}" ]]; then
# Create "adlists.list" by copying "adlists.default" from internal Pi-hole repo
cp "${adListRepoDefault}" "${adListFile}" 2> /dev/null || \
echo -e " ${CROSS} Unable to copy ${adListFile##*/} from ${piholeRepo}"
fi fi
local str="Pulling blocklist source list into range" local str="Pulling blocklist source list into range"
echo -ne " ${INFO} ${str}..." echo -ne " ${INFO} ${str}..."
# Retrieve source URLs from $adListFile # Retrieve source URLs from $adListFile
# Awk Logic: Remove comments (#@;![), CR (windows) line endings and empty lines mapfile -t sources <<< $(
mapfile -t sources < <(awk '!/^[#@;!\[]/ {gsub(/\r$/, "", $0); if ($1) { print $1 } }' "${adListFile}" 2> /dev/null) # Logic: Remove comments (#@;![)
awk '!/^[#@;!\[]/ {
# Remove windows CR line endings
gsub(/\r$/, "", $0)
# Print non-empty line
if ($1) { print $1 }
}' "${adListFile}" 2> /dev/null
)
# Parse source domains from $sources # Parse source domains from $sources
# Awk Logic: Split by folder/port, remove URL protocol & optional username:password@ mapfile -t sourceDomains <<< $(
mapfile -t sourceDomains < <( # Logic: Split by folder/port
awk -F '[/:]' '{ awk -F '[/:]' '{
# Remove URL protocol & optional username:password@
gsub(/(.*:\/\/|.*:.*@)/, "", $0) gsub(/(.*:\/\/|.*:.*@)/, "", $0)
print $1 print $1
}' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null }' <<< "$(printf '%s\n' "${sources[@]}")" 2> /dev/null
@ -151,7 +162,7 @@ gravity_Supernova() {
echo "" echo ""
# Loop through $sources to download each one # Loop through $sources and download each one
for ((i = 0; i < "${#sources[@]}"; i++)); do for ((i = 0; i < "${#sources[@]}"; i++)); do
url="${sources[$i]}" url="${sources[$i]}"
domain="${sourceDomains[$i]}" domain="${sourceDomains[$i]}"
@ -170,11 +181,8 @@ gravity_Supernova() {
esac esac
if [[ "${skipDownload}" == false ]]; then if [[ "${skipDownload}" == false ]]; then
str="Target: ${domain} (${url##*/})" echo -e " ${INFO} Target: ${domain} (${url##*/})"
echo -e " ${INFO} ${str}" gravity_Pull "${url}" "${cmd_ext}" "${agent}"
gravity_Pull "${url}" "${cmd_ext}" "${agent}" "${str}"
echo "" echo ""
fi fi
done done
@ -182,19 +190,15 @@ gravity_Supernova() {
# Download specified URL and perform checks on HTTP status and file content # Download specified URL and perform checks on HTTP status and file content
gravity_Pull() { gravity_Pull() {
local url cmd_ext agent heisenbergCompensator patternBuffer str httpCode success local url="${1}" cmd_ext="${2}" agent="${3}" heisenbergCompensator="" patternBuffer str httpCode success=""
url="${1}" # Create temp file to store content on disk instead of RAM
cmd_ext="${2}" patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb")
agent="${3}"
# Store downloaded content to temp file instead of RAM # Determine if $saveLocation has read permission
patternBuffer=$(mktemp)
heisenbergCompensator=""
if [[ -r "${saveLocation}" ]]; then if [[ -r "${saveLocation}" ]]; then
# Make curl determine if a remote file has been modified since last retrieval # Have curl determine if a remote file has been modified since last retrieval
# Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github links) # Uses "Last-Modified" header, which certain web servers do not provide (e.g: raw github urls)
heisenbergCompensator="-z ${saveLocation}" heisenbergCompensator="-z ${saveLocation}"
fi fi
@ -205,21 +209,21 @@ gravity_Pull() {
# Determine "Status:" output based on HTTP response # Determine "Status:" output based on HTTP response
case "${httpCode}" in case "${httpCode}" in
"200") echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success="true";; "200") echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true;;
"304") echo -e "${OVER} ${TICK} ${str} No changes detected"; success="true";; "304") echo -e "${OVER} ${TICK} ${str} No changes detected"; success=true;;
"000") echo -e "${OVER} ${CROSS} ${str} Connection Refused"; success="false";; "000") echo -e "${OVER} ${CROSS} ${str} Connection Refused";;
"403") echo -e "${OVER} ${CROSS} ${str} Forbidden"; success="false";; "403") echo -e "${OVER} ${CROSS} ${str} Forbidden";;
"404") echo -e "${OVER} ${CROSS} ${str} Not found"; success="false";; "404") echo -e "${OVER} ${CROSS} ${str} Not found";;
"408") echo -e "${OVER} ${CROSS} ${str} Time-out"; success="false";; "408") echo -e "${OVER} ${CROSS} ${str} Time-out";;
"451") echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons"; success="false";; "451") echo -e "${OVER} ${CROSS} ${str} Unavailable For Legal Reasons";;
"521") echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)"; success="false";; "500") echo -e "${OVER} ${CROSS} ${str} Internal Server Error";;
"522") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)"; success="false";; "521") echo -e "${OVER} ${CROSS} ${str} Web Server Is Down (Cloudflare)";;
"500") echo -e "${OVER} ${CROSS} ${str} Internal Server Error"; success="false";; "522") echo -e "${OVER} ${CROSS} ${str} Connection Timed Out (Cloudflare)";;
* ) echo -e "${OVER} ${CROSS} ${str} ${httpCode}"; success="false";; * ) echo -e "${OVER} ${CROSS} ${str} ${httpCode}";;
esac esac
# Determine if the blocklist was downloaded and saved correctly # Determine if the blocklist was downloaded and saved correctly
if [[ "${success}" == "true" ]]; then if [[ "${success}" == true ]]; then
if [[ "${httpCode}" == "304" ]]; then if [[ "${httpCode}" == "304" ]]; then
: # Do not attempt to re-parse file : # Do not attempt to re-parse file
# Check if $patternbuffer is a non-zero length file # Check if $patternbuffer is a non-zero length file
@ -231,46 +235,40 @@ gravity_Pull() {
echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
fi fi
else else
# Determine if cached list exists # Determine if cached list has read permission
if [[ -r "${saveLocation}" ]]; then if [[ -r "${saveLocation}" ]]; then
echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}"
else else
echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}"
fi fi
fi fi
# Delete temp file if it has not been moved
if [[ -f "${patternBuffer}" ]]; then
rm "${patternBuffer}" 2> /dev/null || \
echo -e " ${CROSS} Unable to remove ${patternBuffer}"
fi
} }
# Parse non-standard source files into domains-only format # Parse source files into domains format
gravity_ParseFileIntoDomains() { gravity_ParseFileIntoDomains() {
local source="${1}" destination="${2}" commentPattern firstLine abpFilter local source="${1}" destination="${2}" commentPattern firstLine abpFilter
# Determine how to parse source file # Determine if we are parsing a consolidated list
if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then
# Consolidated list parsing: Remove comments and hosts IP's
# Define symbols used as comments: #;@![/ # Define symbols used as comments: #;@![/
commentPattern="[#;@![\\/]" commentPattern="[#;@![\\/]"
# Awk Logic: Process lines which do not begin with comments # Parse Domains/Hosts files by removing comments & host IPs
# Logic: Ignore lines which begin with comments
awk '!/^'"${commentPattern}"'/ { awk '!/^'"${commentPattern}"'/ {
# If there are multiple words seperated by space # Determine if there are multiple words seperated by a space
if (NF>1) { if(NF>1) {
# Remove comments (including prefixed spaces/tabs) # Remove comments (including prefixed spaces/tabs)
if ($0 ~ /'"${commentPattern}"'/) { gsub("( |\t)'"${commentPattern}"'.*", "", $0) } if($0 ~ /'"${commentPattern}"'/) { gsub("( |\t)'"${commentPattern}"'.*", "", $0) }
# Print consecutive domains # Determine if there are aliased domains
if ($3) { if($3) {
# Remove IP address
$1="" $1=""
# Remove space which is left in $0 when removing $1 # Remove space which is left in $0 when removing $1
gsub("^ ", "", $0) gsub("^ ", "", $0)
print $0 print $0
# Print single domain } else if($2) {
} else if ($2) { # Print single domain without IP
print $2 print $2
} }
# If there are no words seperated by space # If there are no words seperated by space
@ -278,59 +276,60 @@ gravity_ParseFileIntoDomains() {
print $1 print $1
} }
}' "${source}" 2> /dev/null > "${destination}" }' "${source}" 2> /dev/null > "${destination}"
else return 0
# Individual file parsing: Keep comments, while parsing domains from each line fi
read -r firstLine < "${source}"
# Determine how to parse individual source file formats # Individual file parsing: Keep comments, while parsing domains from each line
# Lists may not capitalise the first line correctly, so compare strings against lower case # We keep comments to respect the list maintainer's licensing
if [[ "${firstLine,,}" =~ (adblock|ublock|!checksum) ]]; then read -r firstLine < "${source}"
# Awk Logic: Parse Adblock domains & comments: https://adblockplus.org/filter-cheatsheet
abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/" # Determine how to parse individual source file formats
awk ''"${abpFilter}"' { if [[ "${firstLine,,}" =~ (adblock|ublock|^!) ]]; then
# Remove valid adblock type options # Compare $firstLine against lower case words found in Adblock lists
gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0)
# Remove starting domain name anchor "||" and ending seperator "^$" ($ optional) # Define symbols used as comments: [!
gsub(/(\|\||\^\$?$)/, "", $0) # "||.*^" includes the "Example 2" domains we can extract
# Remove lines which are only IPv4 addresses or contain "^/*" # https://adblockplus.org/filter-cheatsheet
if ($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" } abpFilter="/^(\\[|!)|^(\\|\\|.*\\^)/"
if ($0) { print $0 }
}' "${source}" 2> /dev/null > "${destination}" # Parse Adblock lists by extracting "Example 2" domains
# Parse URL list if source file contains http:// or IPv4 # Logic: Ignore lines which do not include comments or domain name anchor
elif grep -q -E "^(https?://|([0-9]{1,3}\\.){3}[0-9]{1,3}$)" "${source}" &> /dev/null; then awk ''"${abpFilter}"' {
awk '{ # Remove valid adblock type options
# Remove URL protocol, optional "username:password@", and ":?/;" gsub(/~?(important|third-party|popup|subdocument|websocket),?/, "", $0)
if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) } # Remove starting domain name anchor "||" and ending seperator "^$" ($ optional)
# Remove lines which are only IPv4 addresses gsub(/(\|\||\^\$?$)/, "", $0)
if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } # Remove lines which are only IPv4 addresses or contain "^/*"
if ($0) { print $0 } if($0 ~ /(^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$|[\\^\/\*])/) { $0="" }
}' "${source}" 2> /dev/null > "${destination}" if($0) { print $0 }
}' "${source}" 2> /dev/null > "${destination}"
elif grep -q "^address=/" "${source}" &> /dev/null; then
# Parse Dnsmasq format lists # Parse Dnsmasq format lists
elif grep -q "^address=/" "${source}" &> /dev/null; then echo -e " ${CROSS} ${COL_BOLD}dnsmasq${COL_NC} format lists are not supported"
awk -F/ '{ elif grep -q -E "^(https?://|www\\.)" "${source}" &> /dev/null; then
# Print comments # Parse URL list if source file contains "http://" or "www."
if ($0 ~ "#") { # Scanning for "^IPv4$" is too slow with large (1M) lists on low-end hardware
print $0
# Print domains
} else if ($2) {
print $2
}
}' "${source}" 2> /dev/null > "${destination}"
else
# Keep hosts/domains file in same format as it was downloaded
output=$( { mv "${source}" "${destination}"; } 2>&1 )
status="$?"
if [[ "${status}" -ne 0 ]]; then awk '{
echo -e " ${CROSS} Unable to move tmp file to ${piholeDir} # Remove URL protocol, optional "username:password@", and ":?/;"
${output}" if ($0 ~ /[:?\/;]/) { gsub(/(^.*:\/\/(.*:.*@)?|[:?\/;].*)/, "", $0) }
gravity_Cleanup "error" # Remove lines which are only IPv4 addresses
fi if ($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" }
if ($0) { print $0 }
}' "${source}" 2> /dev/null > "${destination}"
else
# Default: Keep hosts/domains file in same format as it was downloaded
output=$( { mv "${source}" "${destination}"; } 2>&1 )
if [[ ! -e "${destination}" ]]; then
echo -e " ${CROSS} Unable to move tmp file to ${piholeDir}
${output}"
gravity_Cleanup "error"
fi fi
fi fi
} }
# Create unfiltered "Matter and Light" consolidated list # Create (unfiltered) "Matter and Light" consolidated list
gravity_Schwarzschild() { gravity_Schwarzschild() {
local str lastLine local str lastLine
@ -340,22 +339,25 @@ gravity_Schwarzschild() {
# Empty $matterAndLight if it already exists, otherwise, create it # Empty $matterAndLight if it already exists, otherwise, create it
: > "${piholeDir}/${matterAndLight}" : > "${piholeDir}/${matterAndLight}"
# Loop through each *.domains file
for i in "${activeDomains[@]}"; do for i in "${activeDomains[@]}"; do
# Only assimilate list if it is available (download might have failed permanently) # Determine if file has read permissions, as download might have failed
if [[ -r "${i}" ]]; then if [[ -r "${i}" ]]; then
# Compile all blacklisted domains into one file and remove CRs # Remove windows CRs from file, and append into $matterAndLight
tr -d '\r' < "${i}" >> "${piholeDir}/${matterAndLight}" tr -d '\r' < "${i}" >> "${piholeDir}/${matterAndLight}"
# Ensure each source blocklist has a final newline # Ensure that the first line of a new list is on a new line
lastLine=$(tail -1 "${piholeDir}/${matterAndLight}") lastLine=$(tail -1 "${piholeDir}/${matterAndLight}")
[[ "${#lastLine}" -gt 0 ]] && echo "" >> "${piholeDir}/${matterAndLight}" if [[ "${#lastLine}" -gt 0 ]]; then
echo "" >> "${piholeDir}/${matterAndLight}"
fi
fi fi
done done
echo -e "${OVER} ${TICK} ${str}" echo -e "${OVER} ${TICK} ${str}"
} }
# Parse unfiltered consolidated blocklist into filtered domains-only format # Parse consolidated list into (filtered, unique) domains-only format
gravity_Filter() { gravity_Filter() {
local str num local str num
@ -365,77 +367,67 @@ gravity_Filter() {
# Parse into hosts file # Parse into hosts file
gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}" gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}"
# Format file line count as currency # Format $parsedMatter line total as currency
num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")") num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")")
echo -e "${OVER} ${TICK} ${str} echo -e "${OVER} ${TICK} ${str}
${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} domains being pulled in by gravity" ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} domains being pulled in by gravity"
gravity_Unique
}
# Sort and remove duplicate blacklisted domains
gravity_Unique() {
local str num
str="Removing duplicate domains" str="Removing duplicate domains"
echo -ne " ${INFO} ${str}..." echo -ne " ${INFO} ${str}..."
sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}" sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}"
echo -e "${OVER} ${TICK} ${str}" echo -e "${OVER} ${TICK} ${str}"
# Format file line count as currency # Format $preEventHorizon line total as currency
num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
echo -e " ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} unique domains trapped in the Event Horizon" echo -e " ${INFO} ${COL_LIGHT_BLUE}${num}${COL_NC} unique domains trapped in the Event Horizon"
} }
# Whitelist blocklist domain sources # Whitelist unique blocklist domain sources
gravity_WhitelistBLD() { gravity_WhitelistBLD() {
local plural str uniqDomains local plural="" str uniqDomains
echo "" echo ""
plural=; [[ "${#sources[*]}" != "1" ]] && plural="s" [[ "${#sources[@]}" -ne 1 ]] && plural="s"
str="Adding blocklist source${plural} to the whitelist" str="Adding blocklist domain source${plural} to the whitelist"
echo -ne " ${INFO} ${str}..." echo -ne " ${INFO} ${str}..."
# Create array of unique $sourceDomains # Create array of unique $sourceDomains
mapfile -t uniqDomains <<< "$(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")")" mapfile -t uniqDomains <<< "$(awk '{ if(!a[$1]++) { print $1 } }' <<< "$(printf '%s\n' "${sourceDomains[@]}")")"
${WHITELIST_COMMAND} -nr -q "${uniqDomains[*]}" > /dev/null # Whitelist $uniqDomains
"${PIHOLE_COMMAND}" -w -nr -q "${uniqDomains[*]}" &> /dev/null
echo -e "${OVER} ${TICK} ${str}" echo -e "${OVER} ${TICK} ${str}"
} }
# Whitelist user-defined domains # Whitelist user-defined domains
gravity_Whitelist() { gravity_Whitelist() {
local plural str num local num plural="" str
# Test existence of whitelist.txt if [[ ! -f "${whitelistFile}" ]]; then
if [[ -f "${whitelistFile}" ]]; then
# Remove anything in whitelist.txt from the Event Horizon
num=$(wc -l < "${whitelistFile}")
plural=; [[ "${num}" -ne 1 ]] && plural="s"
str="Whitelisting ${num} domain${plural}"
echo -ne " ${INFO} ${str}..."
# Print everything from preEventHorizon into whitelistMatter EXCEPT domains in whitelist.txt
grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${whitelistMatter}"
echo -e "${OVER} ${TICK} ${str}"
else
echo -e " ${INFO} Nothing to whitelist!" echo -e " ${INFO} Nothing to whitelist!"
return 0
fi fi
num=$(wc -l < "${whitelistFile}")
[[ "${num}" -ne 1 ]] && plural="s"
str="Whitelisting ${num} domain${plural}"
echo -ne " ${INFO} ${str}..."
# Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile
grep -F -x -v -f "${whitelistFile}" "${piholeDir}/${preEventHorizon}" > "${piholeDir}/${whitelistMatter}"
echo -e "${OVER} ${TICK} ${str}"
} }
# Output count of blacklisted domains and wildcards # Output count of blacklisted domains and wildcards
gravity_ShowBlockCount() { gravity_ShowBlockCount() {
local num plural str local num plural
if [[ -f "${blacklistFile}" ]]; then if [[ -f "${blacklistFile}" ]]; then
num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")") num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
plural=; [[ "${num}" -ne 1 ]] && plural="s" plural=; [[ "${num}" -ne 1 ]] && plural="s"
str="Exact blocked domain${plural}: ${num}" echo -e " ${INFO} Blacklisted ${num} domain${plural}"
echo -e " ${INFO} ${str}"
else
echo -e " ${INFO} Nothing to blacklist!"
fi fi
if [[ -f "${wildcardFile}" ]]; then if [[ -f "${wildcardFile}" ]]; then
@ -445,47 +437,47 @@ gravity_ShowBlockCount() {
num=$(( num/2 )) num=$(( num/2 ))
fi fi
plural=; [[ "${num}" -ne 1 ]] && plural="s" plural=; [[ "${num}" -ne 1 ]] && plural="s"
echo -e " ${INFO} Wildcard blocked domain${plural}: ${num}" echo -e " ${INFO} Wildcard blocked ${num} domain${plural}"
else
echo -e " ${INFO} No wildcards used!"
fi fi
} }
# Parse list of domains into hosts format # Parse list of domains into hosts format
gravity_ParseDomainsIntoHosts() { gravity_ParseDomainsIntoHosts() {
if [[ -n "${IPV4_ADDRESS}" ]] || [[ -n "${IPV6_ADDRESS}" ]]; then awk -v ipv4="$IPV4_ADDRESS" -v ipv6="$IPV6_ADDRESS" '{
# Awk Logic: Remove CR line endings and print IP before domain if IPv4/6 is used # Remove windows CR line endings
awk -v ipv4addr="$IPV4_ADDRESS" -v ipv6addr="$IPV6_ADDRESS" '{ sub(/\r$/, "")
sub(/\r$/, "") # Parse each line as "ipaddr domain"
if(ipv4addr) { print ipv4addr" "$0; } if(ipv6 && ipv4) {
if(ipv6addr) { print ipv6addr" "$0; } print ipv4" "$0"\n"ipv6" "$0
}' >> "${2}" < "${1}" } else if(!ipv6) {
else print ipv4" "$0
echo -e "${OVER} ${CROSS} ${str}" } else {
echo -e " ${COL_LIGHT_RED}No IP addresses found! Please run 'pihole -r' to reconfigure${COL_NC}\\n" print ipv6" "$0
gravity_Cleanup "error" }
fi }' >> "${2}" < "${1}"
} }
# Create "localhost" entries into hosts format # Create "localhost" entries into hosts format
gravity_ParseLocalDomains() { gravity_ParseLocalDomains() {
local hostname local hostname
if [[ -f "/etc/hostname" ]]; then if [[ -s "/etc/hostname" ]]; then
hostname=$(< "/etc/hostname") hostname=$(< "/etc/hostname")
elif command -v hostname &> /dev/null; then elif command -v hostname &> /dev/null; then
hostname=$(hostname -f) hostname=$(hostname -f)
else else
echo -e " ${CROSS} Unable to determine fully qualified domain name of host" echo -e " ${CROSS} Unable to determine fully qualified domain name of host"
return 0
fi fi
echo -e "${hostname}\\npi.hole" > "${localList}.tmp" echo -e "${hostname}\\npi.hole" > "${localList}.tmp"
# Copy the file over as /etc/pihole/local.list so dnsmasq can use it
rm "${localList}" 2> /dev/null || true
gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}"
rm "${localList}.tmp" 2> /dev/null || true
# Add additional local hosts provided by OpenVPN (if available) # Empty $localList if it already exists, otherwise, create it
: > "${localList}"
gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}"
# Add additional LAN hosts provided by OpenVPN (if available)
if [[ -f "${VPNList}" ]]; then if [[ -f "${VPNList}" ]]; then
awk -F, '{printf $2"\t"$1"\n"}' "${VPNList}" >> "${localList}" awk -F, '{printf $2"\t"$1"\n"}' "${VPNList}" >> "${localList}"
fi fi
@ -512,12 +504,14 @@ gravity_ParseBlacklistDomains() {
# Create user-added blacklist entries # Create user-added blacklist entries
gravity_ParseUserDomains() { gravity_ParseUserDomains() {
if [[ -f "${blacklistFile}" ]]; then if [[ ! -f "${blacklistFile}" ]]; then
gravity_ParseDomainsIntoHosts "${blacklistFile}" "${blackList}.tmp" return 0
# Copy the file over as /etc/pihole/black.list so dnsmasq can use it
mv "${blackList}.tmp" "${blackList}" 2> /dev/null || \
echo -e " ${CROSS} Unable to move ${blackList##*/}.tmp to ${piholeDir}"
fi fi
gravity_ParseDomainsIntoHosts "${blacklistFile}" "${blackList}.tmp"
# Copy the file over as /etc/pihole/black.list so dnsmasq can use it
mv "${blackList}.tmp" "${blackList}" 2> /dev/null || \
echo -e " ${CROSS} Unable to move ${blackList##*/}.tmp to ${piholeDir}"
} }
# Trap Ctrl-C # Trap Ctrl-C
@ -525,15 +519,17 @@ gravity_Trap() {
trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
} }
# Clean up after Gravity # Clean up after Gravity upon exit or cancellation
gravity_Cleanup() { gravity_Cleanup() {
local error="${1:-}" local error="${1:-}"
str="Cleaning up stray matter" str="Cleaning up stray matter"
echo -ne " ${INFO} ${str}..." echo -ne " ${INFO} ${str}..."
# Delete tmp content generated by Gravity
rm ${piholeDir}/pihole.*.txt 2> /dev/null rm ${piholeDir}/pihole.*.txt 2> /dev/null
rm ${piholeDir}/*.tmp 2> /dev/null rm ${piholeDir}/*.tmp 2> /dev/null
rm /tmp/*.phgpb 2> /dev/null
# Remove any unused .domains files # Remove any unused .domains files
for file in ${piholeDir}/*.${domainsExtension}; do for file in ${piholeDir}/*.${domainsExtension}; do
@ -576,16 +572,14 @@ for var in "$@"; do
"-sd" | "--skip-download" ) skipDownload=true;; "-sd" | "--skip-download" ) skipDownload=true;;
"-b" | "--blacklist-only" ) listType="blacklist";; "-b" | "--blacklist-only" ) listType="blacklist";;
"-w" | "--whitelist-only" ) listType="whitelist";; "-w" | "--whitelist-only" ) listType="whitelist";;
"-wild" | "--wildcard-only" ) listType="wildcard";; "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";;
esac esac
done done
# Trap Ctrl-C
gravity_Trap gravity_Trap
# Ensure dnsmasq is restarted when modifying wildcards if [[ "${forceDelete:-}" == true ]]; then
[[ "${listType}" == "wildcard" ]] && dnsRestart="restart"
if [[ "${forceDelete}" == true ]]; then
str="Deleting exising list cache" str="Deleting exising list cache"
echo -ne "${INFO} ${str}..." echo -ne "${INFO} ${str}..."
@ -628,7 +622,7 @@ if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "blacklist" ]]; then
gravity_ParseUserDomains gravity_ParseUserDomains
# Perform when downloading blocklists # Perform when downloading blocklists
if [[ ! "${listType}" == "blacklist" ]]; then if [[ ! "${listType:-}" == "blacklist" ]]; then
gravity_ParseLocalDomains gravity_ParseLocalDomains
gravity_ParseBlacklistDomains gravity_ParseBlacklistDomains
fi fi
@ -646,6 +640,6 @@ echo ""
# Determine if DNS has been restarted by this instance of gravity # Determine if DNS has been restarted by this instance of gravity
if [[ -z "${dnsWasOffline:-}" ]]; then if [[ -z "${dnsWasOffline:-}" ]]; then
# Use "force-reload" when restarting dnsmasq for everything but Wildcards # Use "force-reload" when restarting dnsmasq for everything but Wildcards
"${PIHOLE_COMMAND}" restartdns "${dnsRestart:-force-reload}" "${PIHOLE_COMMAND}" restartdns "${dnsRestartType:-force-reload}"
fi fi
"${PIHOLE_COMMAND}" status "${PIHOLE_COMMAND}" status