mirror of
https://github.com/pi-hole/pi-hole.git
synced 2024-11-22 06:03:43 +00:00
Use api.sh within query.sh
Signed-off-by: Christian König <ckoenig@posteo.de>
This commit is contained in:
Christian König
parent
2227a2c569
commit
8e0a40e717
2 changed files with 104 additions and 223 deletions
|
|
@ -1,5 +1,5 @@
|
|||
# Determine if terminal is capable of showing colors
|
||||
if ([[ -t 1 ]] && [[ $(tput colors) -ge 8 ]]) || [[ "${WEBCALL}" ]]; then
|
||||
if ([ -t 1 ] && [ $(tput colors) -ge 8 ]) || [ "${WEBCALL}" ]; then
|
||||
# Bold and underline may not show up on all clients
|
||||
# If something MUST be emphasized, use both
|
||||
COL_BOLD='[1m'
|
||||
|
|
|
|||
331
advanced/Scripts/query.sh
Executable file → Normal file
331
advanced/Scripts/query.sh
Executable file → Normal file
|
|
@ -1,259 +1,140 @@
|
|||
#!/usr/bin/env bash
|
||||
#!/usr/bin/env sh
|
||||
# shellcheck disable=SC1090
|
||||
|
||||
# Ignore warning about `local` being undefinded in POSIX
|
||||
# shellcheck disable=SC3043
|
||||
# https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions
|
||||
|
||||
# Pi-hole: A black hole for Internet advertisements
|
||||
# (c) 2018 Pi-hole, LLC (https://pi-hole.net)
|
||||
# (c) 2023 Pi-hole, LLC (https://pi-hole.net)
|
||||
# Network-wide ad blocking via your own hardware.
|
||||
#
|
||||
# Query Domain Lists
|
||||
# Search Adlists
|
||||
#
|
||||
# This file is copyright under the latest version of the EUPL.
|
||||
# Please see LICENSE file for your rights under this license.
|
||||
|
||||
# Globals
|
||||
piholeDir="/etc/pihole"
|
||||
GRAVITYDB="${piholeDir}/gravity.db"
|
||||
options="$*"
|
||||
all=""
|
||||
exact=""
|
||||
matchType="match"
|
||||
# Source pihole-FTL from install script
|
||||
pihole_FTL="${piholeDir}/pihole-FTL.conf"
|
||||
if [[ -f "${pihole_FTL}" ]]; then
|
||||
source "${pihole_FTL}"
|
||||
fi
|
||||
|
||||
# Set this only after sourcing pihole-FTL.conf as the gravity database path may
|
||||
# have changed
|
||||
gravityDBfile="${GRAVITYDB}"
|
||||
PI_HOLE_INSTALL_DIR="/opt/pihole"
|
||||
max_results="20"
|
||||
partial="true"
|
||||
domain=""
|
||||
|
||||
# Source color table
|
||||
colfile="/opt/pihole/COL_TABLE"
|
||||
source "${colfile}"
|
||||
. "${colfile}"
|
||||
|
||||
if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then
|
||||
# Source api functions
|
||||
. "${PI_HOLE_INSTALL_DIR}/api.sh"
|
||||
|
||||
Help(){
|
||||
echo "Usage: pihole -q [option] <domain>
|
||||
Example: 'pihole -q -exact domain.com'
|
||||
Example: 'pihole -q --exact domain.com'
|
||||
Query the adlists for a specified domain
|
||||
|
||||
Options:
|
||||
-exact Search the adlists for exact domain matches
|
||||
-all Return all query matches within the adlists
|
||||
--exact Search the adlists for exact domain matches
|
||||
--all Return all query matches within the adlists
|
||||
-h, --help Show this help dialog"
|
||||
exit 0
|
||||
}
|
||||
|
||||
|
||||
GenerateOutput(){
|
||||
local data gravity_data lists_data num_gravity num_lists search_type_str
|
||||
local gravity_data_csv lists_data_csv line current_domain
|
||||
data="${1}"
|
||||
|
||||
# construct a new json for the list results where each object contains the domain and the related type
|
||||
lists_data=$(echo "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]')
|
||||
|
||||
# construct a new json for the gravity results where each object contains the adlist URL and the related domains
|
||||
gravity_data=$(echo "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })')
|
||||
|
||||
# number of objects in each json
|
||||
num_gravity=$(echo "${gravity_data}" | jq length )
|
||||
num_lists=$(echo "${lists_data}" | jq length )
|
||||
|
||||
if [ "${partial}" = true ]; then
|
||||
search_type_str="partially"
|
||||
else
|
||||
search_type_str="exactly"
|
||||
fi
|
||||
|
||||
# Handle valid options
|
||||
[[ "${options}" == *"-all"* ]] && all=true
|
||||
if [[ "${options}" == *"-exact"* ]]; then
|
||||
exact="exact"; matchType="exact ${matchType}"
|
||||
# Results from allow/deny list
|
||||
printf "%s\n\n" "Found ${num_lists} domains ${search_type_str} matching '${COL_BLUE}${domain}${COL_NC}'."
|
||||
if [ "${num_lists}" -gt 0 ]; then
|
||||
# Convert the data to a csv, each line is a "domain,type" string
|
||||
# not using jq's @csv here as it quotes each value individually
|
||||
lists_data_csv=$(echo "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")' )
|
||||
|
||||
# Generate output for each csv line, separating line in a domain and type substring at the ','
|
||||
echo "${lists_data_csv}" | while read -r line; do
|
||||
printf "%s\n\n" " - ${COL_GREEN}${line%,*}${COL_NC} (type: exact ${line#*,} domain)"
|
||||
done
|
||||
fi
|
||||
|
||||
# Strip valid options, leaving only the domain and invalid options
|
||||
# This allows users to place the options before or after the domain
|
||||
options=$(sed -E 's/ +-(all|exact) ?//g' <<< "${options}")
|
||||
# Results from gravity
|
||||
printf "%s\n\n" "Found ${num_gravity} adlists ${search_type_str} matching '${COL_BLUE}${domain}${COL_NC}'."
|
||||
if [ "${num_gravity}" -gt 0 ]; then
|
||||
# Convert the data to a csv, each line is a "URL,domain,domain,...." string
|
||||
# not using jq's @csv here as it quotes each value individually
|
||||
gravity_data_csv=$(echo "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")' )
|
||||
|
||||
# Handle remaining options
|
||||
# If $options contain non ASCII characters, convert to punycode
|
||||
case "${options}" in
|
||||
"" ) str="No domain specified";;
|
||||
*" "* ) str="Unknown query option specified";;
|
||||
*[![:ascii:]]* ) rawDomainQuery=$(idn2 "${options}");;
|
||||
* ) rawDomainQuery="${options}";;
|
||||
esac
|
||||
# Generate line-by-line output for each csv line
|
||||
echo "${gravity_data_csv}" | while read -r line; do
|
||||
|
||||
# print adlist URL
|
||||
printf "%s\n\n" " - ${COL_BLUE}${line%%,*}${COL_NC}"
|
||||
|
||||
# cut off URL, leaving "domain,domain,...."
|
||||
line=${line#*,}
|
||||
# print each domain and remove it from the string until nothing is left
|
||||
while [ ${#line} -gt 0 ]; do
|
||||
current_domain=${line%%,*}
|
||||
printf ' - %s\n' "${COL_GREEN}${current_domain}${COL_NC}"
|
||||
# we need to remove the current_domain and the comma in two steps because
|
||||
# the last domain won't have a trailing comma and the while loop wouldn't exit
|
||||
line=${line#"${current_domain}"}
|
||||
line=${line#,}
|
||||
done
|
||||
printf "\n\n"
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
Main(){
|
||||
local data
|
||||
|
||||
if [ -z "${domain}" ]; then
|
||||
echo "No domain specified"; exit 1
|
||||
else
|
||||
# convert domain to punycode
|
||||
domain=$(idn2 "${domain}")
|
||||
|
||||
# convert the domain to lowercase
|
||||
domainQuery=$(echo "${rawDomainQuery}" | tr '[:upper:]' '[:lower:]')
|
||||
|
||||
if [[ -n "${str:-}" ]]; then
|
||||
echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information."
|
||||
exit 1
|
||||
domain=$(echo "${domain}" | tr '[:upper:]' '[:lower:]')
|
||||
fi
|
||||
|
||||
# Scan a domain again a list of RegEX
|
||||
scanRegExList(){
|
||||
local domain="${1}" list="${2}"
|
||||
|
||||
for entry in ${list}; do
|
||||
if [[ "${domain}" =~ ${entry} ]]; then
|
||||
printf "%b\n" "${entry}";
|
||||
fi
|
||||
done
|
||||
# Test if the authentication endpoint is available
|
||||
TestAPIAvailability
|
||||
# Authenticate with the FTL server
|
||||
Authenthication
|
||||
|
||||
data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}")
|
||||
GenerateOutput "${data}"
|
||||
DeleteSession
|
||||
}
|
||||
|
||||
scanDatabaseTable() {
|
||||
local domain table list_type querystr result extra abpquerystr abpfound abpentry searchstr
|
||||
domain="$(printf "%q" "${1}")"
|
||||
table="${2}"
|
||||
list_type="${3:-}"
|
||||
|
||||
# As underscores are legitimate parts of domains, we escape them when using the LIKE operator.
|
||||
# Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this
|
||||
# behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
|
||||
# as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
|
||||
if [[ "${table}" == "gravity" ]]; then
|
||||
|
||||
# Are there ABP entries on gravity?
|
||||
# Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set
|
||||
abpquerystr="SELECT EXISTS (SELECT 1 FROM info WHERE property='abp_domains' and value='1')"
|
||||
abpfound="$(pihole-FTL sqlite3 "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null
|
||||
|
||||
# Create search string for ABP entries only if needed
|
||||
if [ "${abpfound}" -eq 1 ]; then
|
||||
abpentry="${domain}"
|
||||
|
||||
searchstr="'||${abpentry}^'"
|
||||
|
||||
# While a dot is found ...
|
||||
while [ "${abpentry}" != "${abpentry/./}" ]
|
||||
do
|
||||
# ... remove text before the dot (including the dot) and append the result to $searchstr
|
||||
abpentry=$(echo "${abpentry}" | cut -f 2- -d '.')
|
||||
searchstr="$searchstr, '||${abpentry}^'"
|
||||
done
|
||||
|
||||
# The final search string will look like:
|
||||
# "domain IN ('||sub2.sub1.domain.com^', '||sub1.domain.com^', '||domain.com^', '||com^') OR"
|
||||
searchstr="domain IN (${searchstr}) OR "
|
||||
fi
|
||||
|
||||
case "${exact}" in
|
||||
"exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";;
|
||||
* ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE ${searchstr} domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
|
||||
# Process all options (if present)
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
"-h" | "--help" ) Help;;
|
||||
"--exact" ) partial="false";;
|
||||
"--all" ) max_results=10000;; # hard-coded FTL limit
|
||||
* ) domain=$1;;
|
||||
esac
|
||||
else
|
||||
case "${exact}" in
|
||||
"exact" ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain = '${domain}'";;
|
||||
* ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
|
||||
esac
|
||||
fi
|
||||
|
||||
# Send prepared query to gravity database
|
||||
result="$(pihole-FTL sqlite3 -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null
|
||||
if [[ -z "${result}" ]]; then
|
||||
# Return early when there are no matches in this table
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ "${table}" == "gravity" ]]; then
|
||||
echo "${result}"
|
||||
return
|
||||
fi
|
||||
|
||||
# Mark domain as having been white-/blacklist matched (global variable)
|
||||
wbMatch=true
|
||||
|
||||
# Print table name
|
||||
echo " ${matchType^} found in ${COL_BOLD}exact ${table}${COL_NC}"
|
||||
|
||||
# Loop over results and print them
|
||||
mapfile -t results <<< "${result}"
|
||||
for result in "${results[@]}"; do
|
||||
domain="${result/,*}"
|
||||
if [[ "${result#*,}" == "0" ]]; then
|
||||
extra=" (disabled)"
|
||||
else
|
||||
extra=""
|
||||
fi
|
||||
echo " ${domain}${extra}"
|
||||
done
|
||||
}
|
||||
|
||||
scanRegexDatabaseTable() {
|
||||
local domain list list_type
|
||||
domain="${1}"
|
||||
list="${2}"
|
||||
list_type="${3:-}"
|
||||
|
||||
# Query all regex from the corresponding database tables
|
||||
mapfile -t regexList < <(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${list_type}" 2> /dev/null)
|
||||
|
||||
# If we have regexps to process
|
||||
if [[ "${#regexList[@]}" -ne 0 ]]; then
|
||||
# Split regexps over a new line
|
||||
str_regexList=$(printf '%s\n' "${regexList[@]}")
|
||||
# Check domain against regexps
|
||||
mapfile -t regexMatches < <(scanRegExList "${domain}" "${str_regexList}")
|
||||
# If there were regex matches
|
||||
if [[ "${#regexMatches[@]}" -ne 0 ]]; then
|
||||
# Split matching regexps over a new line
|
||||
str_regexMatches=$(printf '%s\n' "${regexMatches[@]}")
|
||||
# Form a "matched" message
|
||||
str_message="${matchType^} found in ${COL_BOLD}regex ${list}${COL_NC}"
|
||||
# Form a "results" message
|
||||
str_result="${COL_BOLD}${str_regexMatches}${COL_NC}"
|
||||
# If we are displaying more than just the source of the block
|
||||
# Set the wildcard match flag
|
||||
wcMatch=true
|
||||
# Echo the "matched" message, indented by one space
|
||||
echo " ${str_message}"
|
||||
# Echo the "results" message, each line indented by three spaces
|
||||
# shellcheck disable=SC2001
|
||||
echo "${str_result}" | sed 's/^/ /'
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
# Scan Whitelist and Blacklist
|
||||
scanDatabaseTable "${domainQuery}" "whitelist" "0"
|
||||
scanDatabaseTable "${domainQuery}" "blacklist" "1"
|
||||
|
||||
# Scan Regex table
|
||||
scanRegexDatabaseTable "${domainQuery}" "whitelist" "2"
|
||||
scanRegexDatabaseTable "${domainQuery}" "blacklist" "3"
|
||||
|
||||
# Query block lists
|
||||
mapfile -t results <<< "$(scanDatabaseTable "${domainQuery}" "gravity")"
|
||||
|
||||
# Handle notices
|
||||
if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then
|
||||
echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the adlists"
|
||||
exit 0
|
||||
elif [[ -z "${results[*]}" ]]; then
|
||||
# Result found in WL/BL/Wildcards
|
||||
exit 0
|
||||
elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then
|
||||
echo -e " ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC}
|
||||
This can be overridden using the -all option"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Print "Exact matches for" title
|
||||
if [[ -n "${exact}" ]]; then
|
||||
plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es"
|
||||
echo " ${matchType^}${plural} for ${COL_BOLD}${domainQuery}${COL_NC} found in:"
|
||||
fi
|
||||
|
||||
for result in "${results[@]}"; do
|
||||
match="${result/,*/}"
|
||||
extra="${result#*,}"
|
||||
adlistAddress="${extra/,*/}"
|
||||
extra="${extra#*,}"
|
||||
if [[ "${extra}" == "0" ]]; then
|
||||
extra=" (disabled)"
|
||||
else
|
||||
extra=""
|
||||
fi
|
||||
|
||||
if [[ -n "${exact}" ]]; then
|
||||
echo " - ${adlistAddress}${extra}"
|
||||
else
|
||||
if [[ ! "${adlistAddress}" == "${adlistAddress_prev:-}" ]]; then
|
||||
count=""
|
||||
echo " ${matchType^} found in ${COL_BOLD}${adlistAddress}${COL_NC}:"
|
||||
adlistAddress_prev="${adlistAddress}"
|
||||
fi
|
||||
: $((count++))
|
||||
|
||||
# Print matching domain if $max_count has not been reached
|
||||
[[ -z "${all}" ]] && max_count="50"
|
||||
if [[ -z "${all}" ]] && [[ "${count}" -ge "${max_count}" ]]; then
|
||||
[[ "${count}" -gt "${max_count}" ]] && continue
|
||||
echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}"
|
||||
else
|
||||
echo " ${match}${extra}"
|
||||
fi
|
||||
fi
|
||||
shift
|
||||
done
|
||||
|
||||
exit 0
|
||||
Main "${domain}"
|
||||
|
|
|
|||
Loading…
Reference in a new issue