Merge branch 'development' into release/v4.3

This commit is contained in:
Dan Schaper 2019-05-18 13:18:14 -07:00 committed by GitHub
commit 7de6bbca61
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 540 additions and 453 deletions

View file

@ -11,46 +11,45 @@
# Globals
basename=pihole
piholeDir=/etc/"${basename}"
whitelist="${piholeDir}"/whitelist.txt
blacklist="${piholeDir}"/blacklist.txt
gravityDBfile="${piholeDir}/gravity.db"
readonly regexlist="/etc/pihole/regex.list"
reload=false
addmode=true
verbose=true
wildcard=false
web=false
domList=()
listMain=""
listAlt=""
listType=""
listname=""
colfile="/opt/pihole/COL_TABLE"
source ${colfile}
helpFunc() {
if [[ "${listMain}" == "${whitelist}" ]]; then
if [[ "${listType}" == "whitelist" ]]; then
param="w"
type="white"
elif [[ "${listMain}" == "${regexlist}" && "${wildcard}" == true ]]; then
type="whitelist"
elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then
param="-wild"
type="wildcard black"
elif [[ "${listMain}" == "${regexlist}" ]]; then
type="wildcard blacklist"
elif [[ "${listType}" == "regex" ]]; then
param="-regex"
type="regex black"
type="regex filter"
else
param="b"
type="black"
type="blacklist"
fi
echo "Usage: pihole -${param} [options] <domain> <domain2 ...>
Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com'
${type^}list one or more domains
${type^} one or more domains
Options:
-d, --delmode Remove domain(s) from the ${type}list
-nr, --noreload Update ${type}list without refreshing dnsmasq
-d, --delmode Remove domain(s) from the ${type}
-nr, --noreload Update ${type} without reloading the DNS server
-q, --quiet Make output less verbose
-h, --help Show this help dialog
-l, --list Display all your ${type}listed domains
@ -73,7 +72,7 @@ HandleOther() {
# Check validity of domain (don't check for regex entries)
if [[ "${#domain}" -le 253 ]]; then
if [[ "${listMain}" == "${regexlist}" && "${wildcard}" == false ]]; then
if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then
validDomain="${domain}"
else
validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
@ -88,175 +87,143 @@ HandleOther() {
fi
}
PoplistFile() {
# Check whitelist file exists, and if not, create it
if [[ ! -f "${whitelist}" ]]; then
touch "${whitelist}"
fi
# Check blacklist file exists, and if not, create it
if [[ ! -f "${blacklist}" ]]; then
touch "${blacklist}"
ProcessDomainList() {
if [[ "${listType}" == "regex" ]]; then
# Regex filter list
listname="regex filters"
else
# Whitelist / Blacklist
listname="${listType}"
fi
for dom in "${domList[@]}"; do
# Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other
# Format domain into regex filter if requested
if [[ "${wildcard}" == true ]]; then
dom="(^|\\.)${dom//\./\\.}$"
fi
# Logic: If addmode then add to desired list and remove from the other;
# if delmode then remove from desired list but do not add to the other
if ${addmode}; then
AddDomain "${dom}" "${listMain}"
RemoveDomain "${dom}" "${listAlt}"
AddDomain "${dom}" "${listType}"
if [[ ! "${listType}" == "regex" ]]; then
RemoveDomain "${dom}" "${listAlt}"
fi
else
RemoveDomain "${dom}" "${listMain}"
RemoveDomain "${dom}" "${listType}"
fi
done
}
AddDomain() {
local domain list num
# Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
domain="$1"
list="$2"
domain=$(EscapeRegexp "$1")
[[ "${list}" == "${whitelist}" ]] && listname="whitelist"
[[ "${list}" == "${blacklist}" ]] && listname="blacklist"
# Is the domain in the list we want to add it to?
num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")"
if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
[[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
[[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
bool=true
# Is the domain in the list we want to add it to?
grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
if [[ "${bool}" == false ]]; then
# Domain not found in the whitelist file, add it!
if [[ "${verbose}" == true ]]; then
echo -e " ${INFO} Adding ${1} to ${listname}..."
fi
reload=true
# Add it to the list we want to add it to
echo "$1" >> "${list}"
else
if [[ "${verbose}" == true ]]; then
echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!"
fi
fi
elif [[ "${list}" == "${regexlist}" ]]; then
[[ -z "${type}" ]] && type="--wildcard-only"
bool=true
domain="${1}"
[[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$"
# Is the domain in the list?
# Search only for exactly matching lines
grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
if [[ "${bool}" == false ]]; then
if [[ "${verbose}" == true ]]; then
echo -e " ${INFO} Adding ${domain} to regex list..."
fi
reload="restart"
echo "$domain" >> "${regexlist}"
else
if [[ "${verbose}" == true ]]; then
echo -e " ${INFO} ${domain} already exists in regex list, no need to add!"
fi
fi
if [[ "${num}" -ne 0 ]]; then
if [[ "${verbose}" == true ]]; then
echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!"
fi
return
fi
# Domain not found in the table, add it!
if [[ "${verbose}" == true ]]; then
echo -e " ${INFO} Adding ${1} to the ${listname}..."
fi
reload=true
# Insert only the domain here. The enabled and date_added fields will be filled
# with their default values (enabled = true, date_added = current timestamp)
sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES ('${domain}');"
}
RemoveDomain() {
local domain list num
# Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
domain="$1"
list="$2"
domain=$(EscapeRegexp "$1")
[[ "${list}" == "${whitelist}" ]] && listname="whitelist"
[[ "${list}" == "${blacklist}" ]] && listname="blacklist"
# Is the domain in the list we want to remove it from?
num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")"
if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
bool=true
[[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
[[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
# Is it in the list? Logic follows that if its whitelisted it should not be blacklisted and vice versa
grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
if [[ "${bool}" == true ]]; then
# Remove it from the other one
echo -e " ${INFO} Removing $1 from ${listname}..."
# /I flag: search case-insensitive
sed -i "/${domain}/Id" "${list}"
reload=true
else
if [[ "${verbose}" == true ]]; then
echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!"
fi
fi
elif [[ "${list}" == "${regexlist}" ]]; then
[[ -z "${type}" ]] && type="--wildcard-only"
domain="${1}"
[[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$"
bool=true
# Is it in the list?
grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
if [[ "${bool}" == true ]]; then
# Remove it from the other one
echo -e " ${INFO} Removing $domain from regex list..."
local lineNumber
lineNumber=$(grep -Fnx "$domain" "${list}" | cut -f1 -d:)
sed -i "${lineNumber}d" "${list}"
reload=true
else
if [[ "${verbose}" == true ]]; then
echo -e " ${INFO} ${domain} does not exist in regex list, no need to remove!"
fi
fi
if [[ "${num}" -eq 0 ]]; then
if [[ "${verbose}" == true ]]; then
echo -e " ${INFO} ${1} does not exist in ${list}, no need to remove!"
fi
return
fi
}
# Update Gravity
Reload() {
echo ""
pihole -g --skip-download "${type:-}"
# Domain found in the table, remove it!
if [[ "${verbose}" == true ]]; then
echo -e " ${INFO} Removing ${1} from the ${listname}..."
fi
reload=true
# Remove it from the current list
sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = '${domain}';"
}
Displaylist() {
if [[ -f ${listMain} ]]; then
if [[ "${listMain}" == "${whitelist}" ]]; then
string="gravity resistant domains"
else
string="domains caught in the sinkhole"
fi
verbose=false
echo -e "Displaying $string:\n"
count=1
while IFS= read -r RD || [ -n "${RD}" ]; do
echo " ${count}: ${RD}"
count=$((count+1))
done < "${listMain}"
local list listname count num_pipes domain enabled status nicedate
listname="${listType}"
data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)"
if [[ -z $data ]]; then
echo -e "Not showing empty ${listname}"
else
echo -e " ${COL_LIGHT_RED}${listMain} does not exist!${COL_NC}"
echo -e "Displaying ${listname}:"
count=1
while IFS= read -r line
do
# Count number of pipes seen in this line
# This is necessary because we can only detect the pipe separating the fields
# from the end backwards as the domain (which is the first field) may contain
# pipe symbols as they are perfectly valid regex filter control characters
num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")"
# Extract domain and enabled status based on the obtained number of pipe characters
domain="$(cut -d'|' -f"-$((num_pipes-1))" <<< "${line}")"
enabled="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")"
datemod="$(cut -d'|' -f"$((num_pipes+1))" <<< "${line}")"
# Translate boolean status into human readable string
if [[ "${enabled}" -eq 1 ]]; then
status="enabled"
else
status="disabled"
fi
# Get nice representation of numerical date stored in database
nicedate=$(date --rfc-2822 -d "@${datemod}")
echo " ${count}: ${domain} (${status}, last modified ${nicedate})"
count=$((count+1))
done <<< "${data}"
fi
exit 0;
}
NukeList() {
if [[ -f "${listMain}" ]]; then
# Back up original list
cp "${listMain}" "${listMain}.bck~"
# Empty out file
echo "" > "${listMain}"
fi
sqlite3 "${gravityDBfile}" "DELETE FROM ${listType};"
}
for var in "$@"; do
case "${var}" in
"-w" | "whitelist" ) listMain="${whitelist}"; listAlt="${blacklist}";;
"-b" | "blacklist" ) listMain="${blacklist}"; listAlt="${whitelist}";;
"--wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;;
"--regex" | "regex" ) listMain="${regexlist}";;
"-w" | "whitelist" ) listType="whitelist"; listAlt="blacklist";;
"-b" | "blacklist" ) listType="blacklist"; listAlt="whitelist";;
"--wild" | "wildcard" ) listType="regex"; wildcard=true;;
"--regex" | "regex" ) listType="regex";;
"-nr"| "--noreload" ) reload=false;;
"-d" | "--delmode" ) addmode=false;;
"-q" | "--quiet" ) verbose=false;;
"-h" | "--help" ) helpFunc;;
"-l" | "--list" ) Displaylist;;
"--nuke" ) NukeList;;
"--web" ) web=true;;
* ) HandleOther "${var}";;
esac
done
@ -267,9 +234,13 @@ if [[ $# = 0 ]]; then
helpFunc
fi
PoplistFile
ProcessDomainList
# Used on web interface
if $web; then
echo "DONE"
fi
if [[ "${reload}" != false ]]; then
# Ensure that "restart" is used for Wildcard updates
Reload "${reload}"
pihole restartdns reload
fi

View file

@ -90,6 +90,7 @@ checkout() {
local path
path="development/${binary}"
echo "development" > /etc/pihole/ftlbranch
chmod 644 /etc/pihole/ftlbranch
elif [[ "${1}" == "master" ]] ; then
# Shortcut to check out master branches
echo -e " ${INFO} Shortcut \"master\" detected - checking out master branches..."
@ -104,6 +105,7 @@ checkout() {
local path
path="master/${binary}"
echo "master" > /etc/pihole/ftlbranch
chmod 644 /etc/pihole/ftlbranch
elif [[ "${1}" == "core" ]] ; then
str="Fetching branches from ${piholeGitUrl}"
echo -ne " ${INFO} $str"
@ -166,6 +168,7 @@ checkout() {
if check_download_exists "$path"; then
echo " ${TICK} Branch ${2} exists"
echo "${2}" > /etc/pihole/ftlbranch
chmod 644 /etc/pihole/ftlbranch
FTLinstall "${binary}"
restart_service pihole-FTL
enable_service pihole-FTL

View file

@ -39,8 +39,9 @@ if [[ "$@" == *"once"* ]]; then
# Note that moving the file is not an option, as
# dnsmasq would happily continue writing into the
# moved file (it will have the same file handler)
cp /var/log/pihole.log /var/log/pihole.log.1
cp -p /var/log/pihole.log /var/log/pihole.log.1
echo " " > /var/log/pihole.log
chmod 644 /var/log/pihole.log
fi
else
# Manual flushing
@ -53,6 +54,7 @@ else
echo " " > /var/log/pihole.log
if [ -f /var/log/pihole.log.1 ]; then
echo " " > /var/log/pihole.log.1
chmod 644 /var/log/pihole.log.1
fi
fi
# Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history)

14
advanced/Scripts/query.sh Normal file → Executable file
View file

@ -11,7 +11,7 @@
# Globals
piholeDir="/etc/pihole"
adListsList="$piholeDir/adlists.list"
gravityDBfile="${piholeDir}/gravity.db"
wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf"
options="$*"
adlist=""
@ -73,11 +73,6 @@ Options:
exit 0
fi
if [[ ! -e "$adListsList" ]]; then
echo -e "${COL_LIGHT_RED}The file $adListsList was not found${COL_NC}"
exit 1
fi
# Handle valid options
if [[ "${options}" == *"-bp"* ]]; then
exact="exact"; blockpage=true
@ -186,11 +181,8 @@ fi
# Get adlist file content as array
if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
for adlistUrl in $(< "${adListsList}"); do
if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then
adlists+=("${adlistUrl}")
fi
done
# Retrieve source URLs from gravity database
mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)"
fi
# Print "Exact matches for" title

View file

@ -51,6 +51,7 @@ if [[ "$2" == "remote" ]]; then
GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")"
echo -n "${GITHUB_CORE_VERSION}" > "${GITHUB_VERSION_FILE}"
chmod 644 "${GITHUB_VERSION_FILE}"
if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then
GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")"
@ -66,6 +67,7 @@ else
CORE_BRANCH="$(get_local_branch /etc/.pihole)"
echo -n "${CORE_BRANCH}" > "${LOCAL_BRANCH_FILE}"
chmod 644 "${LOCAL_BRANCH_FILE}"
if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then
WEB_BRANCH="$(get_local_branch /var/www/html/admin)"
@ -79,6 +81,7 @@ else
CORE_VERSION="$(get_local_version /etc/.pihole)"
echo -n "${CORE_VERSION}" > "${LOCAL_VERSION_FILE}"
chmod 644 "${LOCAL_VERSION_FILE}"
if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then
WEB_VERSION="$(get_local_version /var/www/html/admin)"

View file

@ -17,6 +17,8 @@ readonly FTLconf="/etc/pihole/pihole-FTL.conf"
# 03 -> wildcards
readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf"
readonly gravityDBfile="/etc/pihole/gravity.db"
coltable="/opt/pihole/COL_TABLE"
if [[ -f ${coltable} ]]; then
source ${coltable}
@ -322,6 +324,7 @@ dhcp-option=option:router,${DHCP_ROUTER}
dhcp-leasefile=/etc/pihole/dhcp.leases
#quiet-dhcp
" > "${dhcpconfig}"
chmod 644 "${dhcpconfig}"
if [[ "${PIHOLE_DOMAIN}" != "none" ]]; then
echo "domain=${PIHOLE_DOMAIN}" >> "${dhcpconfig}"
@ -385,19 +388,17 @@ SetWebUILayout() {
}
CustomizeAdLists() {
list="/etc/pihole/adlists.list"
local address
address="${args[3]}"
if [[ "${args[2]}" == "enable" ]]; then
sed -i "\\@${args[3]}@s/^#http/http/g" "${list}"
sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 1 WHERE address = '${address}'"
elif [[ "${args[2]}" == "disable" ]]; then
sed -i "\\@${args[3]}@s/^http/#http/g" "${list}"
sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 0 WHERE address = '${address}'"
elif [[ "${args[2]}" == "add" ]]; then
if [[ $(grep -c "^${args[3]}$" "${list}") -eq 0 ]] ; then
echo "${args[3]}" >> ${list}
fi
sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlists (address) VALUES ('${address}')"
elif [[ "${args[2]}" == "del" ]]; then
var=$(echo "${args[3]}" | sed 's/\//\\\//g')
sed -i "/${var}/Id" "${list}"
sqlite3 "${gravityDBfile}" "DELETE FROM adlists WHERE address = '${address}'"
else
echo "Not permitted"
return 1
@ -541,11 +542,13 @@ addAudit()
do
echo "${var}" >> /etc/pihole/auditlog.list
done
chmod 644 /etc/pihole/auditlog.list
}
clearAudit()
{
echo -n "" > /etc/pihole/auditlog.list
chmod 644 /etc/pihole/auditlog.list
}
SetPrivacyLevel() {