mirror of
https://github.com/pi-hole/pi-hole.git
synced 2025-04-25 08:40:12 +00:00
Merge pull request #2236 from pi-hole/new/regex-lists
Use regex lists instead of wildcards for blocking
This commit is contained in:
Dan Schaper
GitHub
commit
dbc82cfb6a
5 changed files with 106 additions and 68 deletions
|
|
@ -13,10 +13,11 @@ basename=pihole
|
|||
piholeDir=/etc/"${basename}"
|
||||
whitelist="${piholeDir}"/whitelist.txt
|
||||
blacklist="${piholeDir}"/blacklist.txt
|
||||
readonly wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf"
|
||||
readonly regexlist="/etc/pihole/regex.list"
|
||||
reload=false
|
||||
addmode=true
|
||||
verbose=true
|
||||
wildcard=false
|
||||
|
||||
domList=()
|
||||
|
||||
|
|
@ -31,9 +32,9 @@ helpFunc() {
|
|||
if [[ "${listMain}" == "${whitelist}" ]]; then
|
||||
param="w"
|
||||
type="white"
|
||||
elif [[ "${listMain}" == "${wildcardlist}" ]]; then
|
||||
elif [[ "${listMain}" == "${regexlist}" ]]; then
|
||||
param="wild"
|
||||
type="wildcard black"
|
||||
type="regex black"
|
||||
else
|
||||
param="b"
|
||||
type="black"
|
||||
|
|
@ -57,7 +58,8 @@ Options:
|
|||
EscapeRegexp() {
|
||||
# This way we may safely insert an arbitrary
|
||||
# string in our regular expressions
|
||||
# Also remove leading "." if present
|
||||
# This sed is intentionally executed in three steps to ease maintainability
|
||||
# The first sed removes any amount of leading dots
|
||||
echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g"
|
||||
}
|
||||
|
||||
|
|
@ -65,10 +67,14 @@ HandleOther() {
|
|||
# Convert to lowercase
|
||||
domain="${1,,}"
|
||||
|
||||
# Check validity of domain
|
||||
# Check validity of domain (don't check for regex entries)
|
||||
if [[ "${#domain}" -le 253 ]]; then
|
||||
validDomain=$(grep -P "^((-|_)*[a-z\d]((-|_)*[a-z\d])*(-|_)*)(\.(-|_)*([a-z\d]((-|_)*[a-z\d])*))*$" <<< "${domain}") # Valid chars check
|
||||
validDomain=$(grep -P "^[^\.]{1,63}(\.[^\.]{1,63})*$" <<< "${validDomain}") # Length of each label
|
||||
if [[ "${listMain}" == "${regexlist}" && "${wildcard}" == false ]]; then
|
||||
validDomain="${domain}"
|
||||
else
|
||||
validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
|
||||
validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label
|
||||
fi
|
||||
fi
|
||||
|
||||
if [[ -n "${validDomain}" ]]; then
|
||||
|
|
@ -94,9 +100,6 @@ PoplistFile() {
|
|||
if ${addmode}; then
|
||||
AddDomain "${dom}" "${listMain}"
|
||||
RemoveDomain "${dom}" "${listAlt}"
|
||||
if [[ "${listMain}" == "${whitelist}" || "${listMain}" == "${blacklist}" ]]; then
|
||||
RemoveDomain "${dom}" "${wildcardlist}"
|
||||
fi
|
||||
else
|
||||
RemoveDomain "${dom}" "${listMain}"
|
||||
fi
|
||||
|
|
@ -109,7 +112,6 @@ AddDomain() {
|
|||
|
||||
[[ "${list}" == "${whitelist}" ]] && listname="whitelist"
|
||||
[[ "${list}" == "${blacklist}" ]] && listname="blacklist"
|
||||
[[ "${list}" == "${wildcardlist}" ]] && listname="wildcard blacklist"
|
||||
|
||||
if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
|
||||
[[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
|
||||
|
|
@ -121,7 +123,7 @@ AddDomain() {
|
|||
if [[ "${bool}" == false ]]; then
|
||||
# Domain not found in the whitelist file, add it!
|
||||
if [[ "${verbose}" == true ]]; then
|
||||
echo -e " ${INFO} Adding $1 to $listname..."
|
||||
echo -e " ${INFO} Adding ${1} to ${listname}..."
|
||||
fi
|
||||
reload=true
|
||||
# Add it to the list we want to add it to
|
||||
|
|
@ -131,28 +133,26 @@ AddDomain() {
|
|||
echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!"
|
||||
fi
|
||||
fi
|
||||
elif [[ "${list}" == "${wildcardlist}" ]]; then
|
||||
source "${piholeDir}/setupVars.conf"
|
||||
# Remove the /* from the end of the IP addresses
|
||||
IPV4_ADDRESS=${IPV4_ADDRESS%/*}
|
||||
IPV6_ADDRESS=${IPV6_ADDRESS%/*}
|
||||
elif [[ "${list}" == "${regexlist}" ]]; then
|
||||
[[ -z "${type}" ]] && type="--wildcard-only"
|
||||
bool=true
|
||||
domain="${1}"
|
||||
|
||||
[[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain//\./\\.}$"
|
||||
|
||||
# Is the domain in the list?
|
||||
grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false
|
||||
# Search only for exactly matching lines
|
||||
grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
|
||||
|
||||
if [[ "${bool}" == false ]]; then
|
||||
if [[ "${verbose}" == true ]]; then
|
||||
echo -e " ${INFO} Adding $1 to wildcard blacklist..."
|
||||
echo -e " ${INFO} Adding ${domain} to regex list..."
|
||||
fi
|
||||
reload="restart"
|
||||
echo "address=/$1/${IPV4_ADDRESS}" >> "${wildcardlist}"
|
||||
if [[ "${#IPV6_ADDRESS}" > 0 ]]; then
|
||||
echo "address=/$1/${IPV6_ADDRESS}" >> "${wildcardlist}"
|
||||
fi
|
||||
echo "$domain" >> "${regexlist}"
|
||||
else
|
||||
if [[ "${verbose}" == true ]]; then
|
||||
echo -e " ${INFO} ${1} already exists in wildcard blacklist, no need to add!"
|
||||
echo -e " ${INFO} ${domain} already exists in regex list, no need to add!"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
|
@ -164,7 +164,6 @@ RemoveDomain() {
|
|||
|
||||
[[ "${list}" == "${whitelist}" ]] && listname="whitelist"
|
||||
[[ "${list}" == "${blacklist}" ]] && listname="blacklist"
|
||||
[[ "${list}" == "${wildcardlist}" ]] && listname="wildcard blacklist"
|
||||
|
||||
if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
|
||||
bool=true
|
||||
|
|
@ -174,7 +173,7 @@ RemoveDomain() {
|
|||
grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
|
||||
if [[ "${bool}" == true ]]; then
|
||||
# Remove it from the other one
|
||||
echo -e " ${INFO} Removing $1 from $listname..."
|
||||
echo -e " ${INFO} Removing $1 from ${listname}..."
|
||||
# /I flag: search case-insensitive
|
||||
sed -i "/${domain}/Id" "${list}"
|
||||
reload=true
|
||||
|
|
@ -183,20 +182,25 @@ RemoveDomain() {
|
|||
echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!"
|
||||
fi
|
||||
fi
|
||||
elif [[ "${list}" == "${wildcardlist}" ]]; then
|
||||
elif [[ "${list}" == "${regexlist}" ]]; then
|
||||
[[ -z "${type}" ]] && type="--wildcard-only"
|
||||
domain="${1}"
|
||||
|
||||
[[ "${wildcard}" == true ]] && domain="((^)|(\\.))${domain//\./\\.}$"
|
||||
|
||||
bool=true
|
||||
# Is it in the list?
|
||||
grep -e "address=\/${domain}\/" "${wildcardlist}" > /dev/null 2>&1 || bool=false
|
||||
grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
|
||||
if [[ "${bool}" == true ]]; then
|
||||
# Remove it from the other one
|
||||
echo -e " ${INFO} Removing $1 from $listname..."
|
||||
# /I flag: search case-insensitive
|
||||
sed -i "/address=\/${domain}/Id" "${list}"
|
||||
echo -e " ${INFO} Removing $domain from regex list..."
|
||||
local lineNumber
|
||||
lineNumber=$(grep -Fnx "$domain" "${list}" | cut -f1 -d:)
|
||||
sed -i "${lineNumber}d" "${list}"
|
||||
reload=true
|
||||
else
|
||||
if [[ "${verbose}" == true ]]; then
|
||||
echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!"
|
||||
echo -e " ${INFO} ${domain} does not exist in regex list, no need to remove!"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
|
@ -218,7 +222,7 @@ Displaylist() {
|
|||
verbose=false
|
||||
echo -e "Displaying $string:\n"
|
||||
count=1
|
||||
while IFS= read -r RD; do
|
||||
while IFS= read -r RD || [ -n "${RD}" ]; do
|
||||
echo " ${count}: ${RD}"
|
||||
count=$((count+1))
|
||||
done < "${listMain}"
|
||||
|
|
@ -241,7 +245,8 @@ for var in "$@"; do
|
|||
case "${var}" in
|
||||
"-w" | "whitelist" ) listMain="${whitelist}"; listAlt="${blacklist}";;
|
||||
"-b" | "blacklist" ) listMain="${blacklist}"; listAlt="${whitelist}";;
|
||||
"-wild" | "wildcard" ) listMain="${wildcardlist}";;
|
||||
"--wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;;
|
||||
"--regex" | "regex" ) listMain="${regexlist}";;
|
||||
"-nr"| "--noreload" ) reload=false;;
|
||||
"-d" | "--delmode" ) addmode=false;;
|
||||
"-q" | "--quiet" ) verbose=false;;
|
||||
|
|
|
|||
28
advanced/Scripts/wildcard_regex_converter.sh
Normal file
28
advanced/Scripts/wildcard_regex_converter.sh
Normal file
|
|
@ -0,0 +1,28 @@
|
|||
#!/bin/bash
|
||||
# Pi-hole: A black hole for Internet advertisements
|
||||
# (c) 2017 Pi-hole, LLC (https://pi-hole.net)
|
||||
# Network-wide ad blocking via your own hardware.
|
||||
#
|
||||
# Provides an automated migration subroutine to convert Pi-hole v3.x wildcard domains to Pi-hole v4.x regex filters
|
||||
#
|
||||
# This file is copyright under the latest version of the EUPL.
|
||||
# Please see LICENSE file for your rights under this license.
|
||||
|
||||
# regexFile set in gravity.sh
|
||||
|
||||
wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf"
|
||||
|
||||
convert_wildcard_to_regex() {
|
||||
if [ ! -f "${wildcardFile}" ]; then
|
||||
return
|
||||
fi
|
||||
local addrlines domains uniquedomains
|
||||
# Obtain wildcard domains from old file
|
||||
addrlines="$(grep -oE "/.*/" ${wildcardFile})"
|
||||
# Strip "/" from domain names and convert "." to regex-compatible "\."
|
||||
domains="$(sed 's/\///g;s/\./\\./g' <<< "${addrlines}")"
|
||||
# Remove repeated domains (may have been inserted two times due to A and AAAA blocking)
|
||||
uniquedomains="$(uniq <<< "${domains}")"
|
||||
# Automatically generate regex filters and remove old wildcards file
|
||||
awk '{print "((^)|(\\.))"$0"$"}' <<< "${uniquedomains}" >> "${regexFile:?}" && rm "${wildcardFile}"
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue