Update query.sh to use FTL's API instead of directly interacting with the database (#5361)

This commit is contained in:
yubiuser 2023-10-16 21:00:42 +02:00 committed by GitHub
commit aaa9acc0b9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 270 additions and 223 deletions

View file

@ -1,5 +1,5 @@
# Determine if terminal is capable of showing colors
if ([[ -t 1 ]] && [[ $(tput colors) -ge 8 ]]) || [[ "${WEBCALL}" ]]; then
if ([ -t 1 ] && [ $(tput colors) -ge 8 ]) || [ "${WEBCALL}" ]; then
# Bold and underline may not show up on all clients
# If something MUST be emphasized, use both
COL_BOLD=''

155
advanced/Scripts/api.sh Executable file
View file

@ -0,0 +1,155 @@
#!/usr/bin/env sh
# shellcheck disable=SC3043 #https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions
# Pi-hole: A black hole for Internet advertisements
# (c) 2017 Pi-hole, LLC (https://pi-hole.net)
# Network-wide ad blocking via your own hardware.
#
# Script to hold api functions for use in other scripts
#
# This file is copyright under the latest version of the EUPL.
# Please see LICENSE file for your rights under this license.
# The basic usage steps are
# 1) Test Availability of the API
# 2) Try to authenticate (read password if needed)
# 3) Get the data from the API endpoint
# 4) Delete the session
TestAPIAvailability() {
# as we are running locally, we can get the port value from FTL directly
PORT="$(pihole-FTL --config webserver.port)"
PORT="${PORT%%,*}"
availabilityResonse=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${PORT}/api/auth")
# test if http status code was 200 (OK) or 401 (authentication required)
if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then
echo "API not available at: http://localhost:${PORT}/api"
echo "Exiting."
exit 1
fi
}
Authenthication() {
# Try to authenticate
LoginAPI
while [ "${validSession}" = false ] || [ -z "${validSession}" ] ; do
echo "Authentication failed. Please enter your Pi-hole password"
# secretly read the password
secretRead; printf '\n'
# Try to authenticate again
LoginAPI
done
# Loop exited, authentication was successful
echo "Authentication successful."
}
LoginAPI() {
sessionResponse="$(curl --silent -X POST "http://localhost:${PORT}/api/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )"
if [ -z "${sessionResponse}" ]; then
echo "No response from FTL server. Please check connectivity"
exit 1
fi
# obtain validity and session ID from session response
validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null)
SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null)
}
DeleteSession() {
# if a valid Session exists (no password required or successful authenthication) and
# SID is not null (successful authenthication only), delete the session
if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then
# Try to delete the session. Omit the output, but get the http status code
deleteResponse=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "http://localhost:${PORT}/api/auth" -H "Accept: application/json" -H "sid: ${SID}")
case "${deleteResponse}" in
"200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";;
"401") printf "%b" "Logout attempt without a valid session. Unauthorized!\n";;
"410") printf "%b" "Session successfully deleted.\n";;
esac;
fi
}
GetFTLData() {
local data response status
# get the data from querying the API as well as the http status code
response=$(curl -s -w "%{http_code}" -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" )
# status are the last 3 characters
status=$(printf %s "${response#"${response%???}"}")
# data is everything from response without the last 3 characters
data=$(printf %s "${response%???}")
if [ "${status}" = 200 ]; then
# response OK
echo "${data}"
elif [ "${status}" = 000 ]; then
# connection lost
echo "000"
elif [ "${status}" = 401 ]; then
# unauthorized
echo "401"
fi
}
secretRead() {
# POSIX compliant function to read user-input and
# mask every character entered by (*)
#
# This is challenging, because in POSIX, `read` does not support
# `-s` option (suppressing the input) or
# `-n` option (reading n chars)
# This workaround changes the terminal characteristics to not echo input and later resets this option
# credits https://stackoverflow.com/a/4316765
# showing asterisk instead of password
# https://stackoverflow.com/a/24600839
# https://unix.stackexchange.com/a/464963
# Save current terminal settings (needed for later restore after password prompt)
stty_orig=$(stty -g)
stty -echo # do not echo user input
stty -icanon min 1 time 0 # disable canonical mode https://man7.org/linux/man-pages/man3/termios.3.html
unset password
unset key
unset charcount
charcount=0
while key=$(dd ibs=1 count=1 2>/dev/null); do #read one byte of input
if [ "${key}" = "$(printf '\0' | tr -d '\0')" ] ; then
# Enter - accept password
break
fi
if [ "${key}" = "$(printf '\177')" ] ; then
# Backspace
if [ $charcount -gt 0 ] ; then
charcount=$((charcount-1))
printf '\b \b'
password="${password%?}"
fi
else
# any other character
charcount=$((charcount+1))
printf '*'
password="$password$key"
fi
done
# restore original terminal settings
stty "${stty_orig}"
}

View file

@ -1,259 +1,151 @@
#!/usr/bin/env bash
#!/usr/bin/env sh
# shellcheck disable=SC1090
# Ignore warning about `local` being undefinded in POSIX
# shellcheck disable=SC3043
# https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions
# Pi-hole: A black hole for Internet advertisements
# (c) 2018 Pi-hole, LLC (https://pi-hole.net)
# (c) 2023 Pi-hole, LLC (https://pi-hole.net)
# Network-wide ad blocking via your own hardware.
#
# Query Domain Lists
# Search Adlists
#
# This file is copyright under the latest version of the EUPL.
# Please see LICENSE file for your rights under this license.
# Globals
piholeDir="/etc/pihole"
GRAVITYDB="${piholeDir}/gravity.db"
options="$*"
all=""
exact=""
matchType="match"
# Source pihole-FTL from install script
pihole_FTL="${piholeDir}/pihole-FTL.conf"
if [[ -f "${pihole_FTL}" ]]; then
source "${pihole_FTL}"
fi
# Set this only after sourcing pihole-FTL.conf as the gravity database path may
# have changed
gravityDBfile="${GRAVITYDB}"
PI_HOLE_INSTALL_DIR="/opt/pihole"
max_results="20"
partial="false"
domain=""
# Source color table
colfile="/opt/pihole/COL_TABLE"
source "${colfile}"
. "${colfile}"
if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then
# Source api functions
. "${PI_HOLE_INSTALL_DIR}/api.sh"
Help(){
echo "Usage: pihole -q [option] <domain>
Example: 'pihole -q -exact domain.com'
Example: 'pihole -q --partial domain.com'
Query the adlists for a specified domain
Options:
-exact Search the adlists for exact domain matches
-all Return all query matches within the adlists
--partial Search the adlists for partially matching domains
--all Return all query matches within the adlists
-h, --help Show this help dialog"
exit 0
}
GenerateOutput(){
local data gravity_data lists_data num_gravity num_lists search_type_str
local gravity_data_csv lists_data_csv line current_domain
data="${1}"
# construct a new json for the list results where each object contains the domain and the related type
lists_data=$(echo "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]')
# construct a new json for the gravity results where each object contains the adlist URL and the related domains
gravity_data=$(echo "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })')
# number of objects in each json
num_gravity=$(echo "${gravity_data}" | jq length )
num_lists=$(echo "${lists_data}" | jq length )
if [ "${partial}" = true ]; then
search_type_str="partially"
else
search_type_str="exactly"
fi
# Handle valid options
[[ "${options}" == *"-all"* ]] && all=true
if [[ "${options}" == *"-exact"* ]]; then
exact="exact"; matchType="exact ${matchType}"
# Results from allow/deny list
printf "%s\n\n" "Found ${num_lists} domains ${search_type_str} matching '${COL_BLUE}${domain}${COL_NC}'."
if [ "${num_lists}" -gt 0 ]; then
# Convert the data to a csv, each line is a "domain,type" string
# not using jq's @csv here as it quotes each value individually
lists_data_csv=$(echo "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")' )
# Generate output for each csv line, separating line in a domain and type substring at the ','
echo "${lists_data_csv}" | while read -r line; do
printf "%s\n\n" " - ${COL_GREEN}${line%,*}${COL_NC} (type: exact ${line#*,} domain)"
done
fi
# Strip valid options, leaving only the domain and invalid options
# This allows users to place the options before or after the domain
options=$(sed -E 's/ +-(all|exact) ?//g' <<< "${options}")
# Results from gravity
printf "%s\n\n" "Found ${num_gravity} adlists ${search_type_str} matching '${COL_BLUE}${domain}${COL_NC}'."
if [ "${num_gravity}" -gt 0 ]; then
# Convert the data to a csv, each line is a "URL,domain,domain,...." string
# not using jq's @csv here as it quotes each value individually
gravity_data_csv=$(echo "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")' )
# Handle remaining options
# If $options contain non ASCII characters, convert to punycode
case "${options}" in
"" ) str="No domain specified";;
*" "* ) str="Unknown query option specified";;
*[![:ascii:]]* ) rawDomainQuery=$(idn2 "${options}");;
* ) rawDomainQuery="${options}";;
esac
# Generate line-by-line output for each csv line
echo "${gravity_data_csv}" | while read -r line; do
# print adlist URL
printf "%s\n\n" " - ${COL_BLUE}${line%%,*}${COL_NC}"
# cut off URL, leaving "domain,domain,...."
line=${line#*,}
# print each domain and remove it from the string until nothing is left
while [ ${#line} -gt 0 ]; do
current_domain=${line%%,*}
printf ' - %s\n' "${COL_GREEN}${current_domain}${COL_NC}"
# we need to remove the current_domain and the comma in two steps because
# the last domain won't have a trailing comma and the while loop wouldn't exit
line=${line#"${current_domain}"}
line=${line#,}
done
printf "\n\n"
done
fi
}
Main(){
local data
if [ -z "${domain}" ]; then
echo "No domain specified"; exit 1
else
# convert domain to punycode
domain=$(idn2 "${domain}")
# convert the domain to lowercase
domainQuery=$(echo "${rawDomainQuery}" | tr '[:upper:]' '[:lower:]')
if [[ -n "${str:-}" ]]; then
echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information."
exit 1
domain=$(echo "${domain}" | tr '[:upper:]' '[:lower:]')
fi
# Scan a domain again a list of RegEX
scanRegExList(){
local domain="${1}" list="${2}"
# Test if the authentication endpoint is available
TestAPIAvailability
for entry in ${list}; do
if [[ "${domain}" =~ ${entry} ]]; then
printf "%b\n" "${entry}";
# Users can configure FTL in a way, that for accessing a) all endpoints (webserver.api.localAPIauth)
# or b) for the /search endpoint (webserver.api.searchAPIauth) no authentication is required.
# Therefore, we try to query directly without authentication but do authenticat if 401 is returned
data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}")
if [ "${data}" = 401 ]; then
# Unauthenticated, so authenticate with the FTL server required
Authenthication
# send query again
data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}")
fi
done
GenerateOutput "${data}"
DeleteSession
}
scanDatabaseTable() {
local domain table list_type querystr result extra abpquerystr abpfound abpentry searchstr
domain="$(printf "%q" "${1}")"
table="${2}"
list_type="${3:-}"
# As underscores are legitimate parts of domains, we escape them when using the LIKE operator.
# Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this
# behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
# as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
if [[ "${table}" == "gravity" ]]; then
# Are there ABP entries on gravity?
# Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set
abpquerystr="SELECT EXISTS (SELECT 1 FROM info WHERE property='abp_domains' and value='1')"
abpfound="$(pihole-FTL sqlite3 "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null
# Create search string for ABP entries only if needed
if [ "${abpfound}" -eq 1 ]; then
abpentry="${domain}"
searchstr="'||${abpentry}^'"
# While a dot is found ...
while [ "${abpentry}" != "${abpentry/./}" ]
do
# ... remove text before the dot (including the dot) and append the result to $searchstr
abpentry=$(echo "${abpentry}" | cut -f 2- -d '.')
searchstr="$searchstr, '||${abpentry}^'"
done
# The final search string will look like:
# "domain IN ('||sub2.sub1.domain.com^', '||sub1.domain.com^', '||domain.com^', '||com^') OR"
searchstr="domain IN (${searchstr}) OR "
fi
case "${exact}" in
"exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";;
* ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE ${searchstr} domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
# Process all options (if present)
while [ "$#" -gt 0 ]; do
case "$1" in
"-h" | "--help" ) Help;;
"--partial" ) partial="true";;
"--all" ) max_results=10000;; # hard-coded FTL limit
* ) domain=$1;;
esac
else
case "${exact}" in
"exact" ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain = '${domain}'";;
* ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
esac
fi
# Send prepared query to gravity database
result="$(pihole-FTL sqlite3 -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null
if [[ -z "${result}" ]]; then
# Return early when there are no matches in this table
return
fi
if [[ "${table}" == "gravity" ]]; then
echo "${result}"
return
fi
# Mark domain as having been white-/blacklist matched (global variable)
wbMatch=true
# Print table name
echo " ${matchType^} found in ${COL_BOLD}exact ${table}${COL_NC}"
# Loop over results and print them
mapfile -t results <<< "${result}"
for result in "${results[@]}"; do
domain="${result/,*}"
if [[ "${result#*,}" == "0" ]]; then
extra=" (disabled)"
else
extra=""
fi
echo " ${domain}${extra}"
done
}
scanRegexDatabaseTable() {
local domain list list_type
domain="${1}"
list="${2}"
list_type="${3:-}"
# Query all regex from the corresponding database tables
mapfile -t regexList < <(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${list_type}" 2> /dev/null)
# If we have regexps to process
if [[ "${#regexList[@]}" -ne 0 ]]; then
# Split regexps over a new line
str_regexList=$(printf '%s\n' "${regexList[@]}")
# Check domain against regexps
mapfile -t regexMatches < <(scanRegExList "${domain}" "${str_regexList}")
# If there were regex matches
if [[ "${#regexMatches[@]}" -ne 0 ]]; then
# Split matching regexps over a new line
str_regexMatches=$(printf '%s\n' "${regexMatches[@]}")
# Form a "matched" message
str_message="${matchType^} found in ${COL_BOLD}regex ${list}${COL_NC}"
# Form a "results" message
str_result="${COL_BOLD}${str_regexMatches}${COL_NC}"
# If we are displaying more than just the source of the block
# Set the wildcard match flag
wcMatch=true
# Echo the "matched" message, indented by one space
echo " ${str_message}"
# Echo the "results" message, each line indented by three spaces
# shellcheck disable=SC2001
echo "${str_result}" | sed 's/^/ /'
fi
fi
}
# Scan Whitelist and Blacklist
scanDatabaseTable "${domainQuery}" "whitelist" "0"
scanDatabaseTable "${domainQuery}" "blacklist" "1"
# Scan Regex table
scanRegexDatabaseTable "${domainQuery}" "whitelist" "2"
scanRegexDatabaseTable "${domainQuery}" "blacklist" "3"
# Query block lists
mapfile -t results <<< "$(scanDatabaseTable "${domainQuery}" "gravity")"
# Handle notices
if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then
echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the adlists"
exit 0
elif [[ -z "${results[*]}" ]]; then
# Result found in WL/BL/Wildcards
exit 0
elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then
echo -e " ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC}
This can be overridden using the -all option"
exit 0
fi
# Print "Exact matches for" title
if [[ -n "${exact}" ]]; then
plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es"
echo " ${matchType^}${plural} for ${COL_BOLD}${domainQuery}${COL_NC} found in:"
fi
for result in "${results[@]}"; do
match="${result/,*/}"
extra="${result#*,}"
adlistAddress="${extra/,*/}"
extra="${extra#*,}"
if [[ "${extra}" == "0" ]]; then
extra=" (disabled)"
else
extra=""
fi
if [[ -n "${exact}" ]]; then
echo " - ${adlistAddress}${extra}"
else
if [[ ! "${adlistAddress}" == "${adlistAddress_prev:-}" ]]; then
count=""
echo " ${matchType^} found in ${COL_BOLD}${adlistAddress}${COL_NC}:"
adlistAddress_prev="${adlistAddress}"
fi
: $((count++))
# Print matching domain if $max_count has not been reached
[[ -z "${all}" ]] && max_count="50"
if [[ -z "${all}" ]] && [[ "${count}" -ge "${max_count}" ]]; then
[[ "${count}" -gt "${max_count}" ]] && continue
echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}"
else
echo " ${match}${extra}"
fi
fi
shift
done
exit 0
Main "${domain}"