From bfc18f8329ad6eca2d4c2e3ea9e641d86844a453 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 23:04:39 +0200 Subject: [PATCH] Rewrite list functions to use the API Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 18 +- advanced/Scripts/list.sh | 358 +++++++++++++++------------------------ pihole | 20 +-- 3 files changed, 164 insertions(+), 232 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 4162eff1..21447105 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -75,12 +75,16 @@ TestAPIAvailability() { } LoginAPI() { + if [ -z "${API_URL}" ]; then + TestAPIAvailability + fi + # Try to read the CLI password (if enabled and readable by the current user) if [ -r /etc/pihole/cli_pw ]; then password=$(cat /etc/pihole/cli_pw) # Try to authenticate using the CLI password - LoginAPI + Authentication fi # If this did not work, ask the user for the password @@ -91,7 +95,7 @@ LoginAPI() { secretRead; printf '\n' # Try to authenticate again - LoginAPI + Authentication done } @@ -144,6 +148,16 @@ GetFTLData() { fi } +PostFTLData() { + local data response status + # send the data to the API + response=$(curl -skS -w "%{http_code}" -X POST "${API_URL}$1" --data-raw "$2" -H "Accept: application/json" -H "sid: ${SID}" ) + # status are the last 3 characters + status=$(printf %s "${response#"${response%???}"}") + # data is everything from response without the last 3 characters + printf %s "${response%???}" +} + secretRead() { # POSIX compliant function to read user-input and diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 76558e58..3bd4af75 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -5,261 +5,187 @@ # (c) 2017 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. # -# Whitelist and blacklist domains +# allowlist and denylist domains # # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -# Globals -piholeDir="/etc/pihole" -GRAVITYDB="${piholeDir}/gravity.db" -# Source pihole-FTL from install script -pihole_FTL="${piholeDir}/pihole-FTL.conf" -if [[ -f "${pihole_FTL}" ]]; then - source "${pihole_FTL}" +readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" +readonly utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +source "${utilsfile}" + +readonly apifile="${PI_HOLE_SCRIPT_DIR}/api.sh" +source "${apifile}" + +# Determine database location +DBFILE=$(getFTLConfigValue "files.database") +if [ -z "$DBFILE" ]; then + DBFILE="/etc/pihole/pihole-FTL.db" fi -# Set this only after sourcing pihole-FTL.conf as the gravity database path may -# have changed -gravityDBfile="${GRAVITYDB}" +# Determine gravity database location +GRAVITYDB=$(getFTLConfigValue "files.gravity") +if [ -z "$GRAVITYDB" ]; then + GRAVITYDB="/etc/pihole/gravity.db" +fi -noReloadRequested=false addmode=true verbose=true wildcard=false -web=false domList=() typeId="" comment="" -declare -i domaincount -domaincount=0 -reload=false colfile="/opt/pihole/COL_TABLE" source ${colfile} -# IDs are hard-wired to domain interpretation in the gravity database scheme -# Clients (including FTL) will read them through the corresponding views -readonly whitelist="0" -readonly blacklist="1" -readonly regex_whitelist="2" -readonly regex_blacklist="3" - -GetListnameFromTypeId() { - if [[ "$1" == "${whitelist}" ]]; then - echo "whitelist" - elif [[ "$1" == "${blacklist}" ]]; then - echo "blacklist" - elif [[ "$1" == "${regex_whitelist}" ]]; then - echo "regex whitelist" - elif [[ "$1" == "${regex_blacklist}" ]]; then - echo "regex blacklist" - fi -} - -GetListParamFromTypeId() { - if [[ "${typeId}" == "${whitelist}" ]]; then - echo "w" - elif [[ "${typeId}" == "${blacklist}" ]]; then - echo "b" - elif [[ "${typeId}" == "${regex_whitelist}" && "${wildcard}" == true ]]; then - echo "-white-wild" - elif [[ "${typeId}" == "${regex_whitelist}" ]]; then - echo "-white-regex" - elif [[ "${typeId}" == "${regex_blacklist}" && "${wildcard}" == true ]]; then - echo "-wild" - elif [[ "${typeId}" == "${regex_blacklist}" ]]; then - echo "-regex" - fi -} - helpFunc() { - local listname param - - listname="$(GetListnameFromTypeId "${typeId}")" - param="$(GetListParamFromTypeId)" - - echo "Usage: pihole -${param} [options] -Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com' -${listname^} one or more domains + echo "Usage: pihole ${abbrv} [options] +Example: 'pihole ${abbrv} site.com', or 'pihole ${abbrv} site1.com site2.com' +${typeId^} one or more ${kindId} domains Options: - -d, --delmode Remove domain(s) from the ${listname} - -nr, --noreload Update ${listname} without reloading the DNS server + -d, --delmode Remove domain(s) -q, --quiet Make output less verbose -h, --help Show this help dialog - -l, --list Display all your ${listname}listed domains + -l, --list Display domains --nuke Removes all entries in a list --comment \"text\" Add a comment to the domain. If adding multiple domains the same comment will be used for all" exit 0 } -ValidateDomain() { - # Convert to lowercase - domain="${1,,}" - local str validDomain - - # Check validity of domain (don't check for regex entries) - if [[ ( "${typeId}" == "${regex_blacklist}" || "${typeId}" == "${regex_whitelist}" ) && "${wildcard}" == false ]]; then - validDomain="${domain}" - else - # Check max length - if [[ "${#domain}" -le 253 ]]; then - validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check - validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label - # set error string - str="is not a valid argument or domain name!" - else - validDomain= - str="is too long!" - - fi +CreateDomainList() { + # Format domain into regex filter if requested + local dom=${1} + if [[ "${wildcard}" == true ]]; then + dom="(\\.|^)${dom//\./\\.}$" fi - - if [[ -n "${validDomain}" ]]; then - domList=("${domList[@]}" "${validDomain}") - else - echo -e " ${CROSS} ${domain} ${str}" - fi - - domaincount=$((domaincount+1)) -} - -ProcessDomainList() { - for dom in "${domList[@]}"; do - # Format domain into regex filter if requested - if [[ "${wildcard}" == true ]]; then - dom="(\\.|^)${dom//\./\\.}$" - fi - - # Logic: If addmode then add to desired list and remove from the other; - # if delmode then remove from desired list but do not add to the other - if ${addmode}; then - AddDomain "${dom}" - else - RemoveDomain "${dom}" - fi - done + domList=("${domList[@]}" "${dom}") } AddDomain() { - local domain num requestedListname existingTypeId existingListname - domain="$1" + local json num - # Is the domain in the list we want to add it to? - num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" - requestedListname="$(GetListnameFromTypeId "${typeId}")" + # Authenticate with the API + LoginAPI - if [[ "${num}" -ne 0 ]]; then - existingTypeId="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" - if [[ "${existingTypeId}" == "${typeId}" ]]; then - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${requestedListname}, no need to add!" + # Prepare request to POST /api/domains/{type}/{kind} + # Build JSON object of the following form + # { + # "domain": [ ], + # "comment": + # } + # where is an array of domain strings and is a string + # We use jq to build the JSON object + json=$(jq --null-input --compact-output --arg domains "${domList[*]}" --arg comment "${comment}" '{domain: $domains | split(" "), comment: $comment}') + + # Send the request + data=$(PostFTLData "domains/${typeId}/${kindId}" "${json}") + + # Display domain(s) added + # (they are listed in .processed.success, use jq) + num=$(echo "${data}" | jq '.processed.success | length') + if [[ "${num}" -gt 0 ]] && [[ "${verbose}" == true ]]; then + echo -e " ${TICK} Added ${num} domain(s):" + for i in $(seq 0 $((num-1))); do + echo -e " - ${COL_BLUE}$(echo "${data}" | jq --raw-output ".processed.success[$i].item")${COL_NC}" + done + fi + # Display failed domain(s) + # (they are listed in .processed.errors, use jq) + num=$(echo "${data}" | jq '.processed.errors | length') + if [[ "${num}" -gt 0 ]] && [[ "${verbose}" == true ]]; then + echo -e " ${CROSS} Failed to add ${num} domain(s):" + for i in $(seq 0 $((num-1))); do + echo -e " - ${COL_BLUE}$(echo "${data}" | jq --raw-output ".processed.errors[$i].item")${COL_NC}" + error=$(echo "${data}" | jq --raw-output ".processed.errors[$i].error") + if [[ "${error}" == "UNIQUE constraint failed: domainlist.domain, domainlist.type" ]]; then + error="Domain already in the specified list" fi - else - existingListname="$(GetListnameFromTypeId "${existingTypeId}")" - pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!" - fi - fi - return + echo -e " ${error}" + done fi - # Domain not found in the table, add it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${domain} to the ${requestedListname}..." - fi - reload=true - # Insert only the domain here. The enabled and date_added fields will be filled - # with their default values (enabled = true, date_added = current timestamp) - if [[ -z "${comment}" ]]; then - pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" - else - # also add comment when variable has been set through the "--comment" option - pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" - fi + # Log out + LogoutAPI } RemoveDomain() { - local domain num requestedListname - domain="$1" + local json num - # Is the domain in the list we want to remove it from? - num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" + # Authenticate with the API + LoginAPI - requestedListname="$(GetListnameFromTypeId "${typeId}")" + # Prepare request to POST /api/domains:batchDelete + # Build JSON object of the following form + # [{ + # "item": , + # "type": "${typeId}", + # "kind": "${kindId}", + # }] + # where is the domain string and ${typeId} and ${kindId} are the type and kind IDs + # We use jq to build the JSON object) + json=$(jq --null-input --compact-output --arg domains "${domList[*]}" --arg typeId "${typeId}" --arg kindId "${kindId}" '[ $domains | split(" ")[] as $item | {item: $item, type: $typeId, kind: $kindId} ]') - if [[ "${num}" -eq 0 ]]; then - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${domain} does not exist in ${requestedListname}, no need to remove!" - fi - return + # Send the request + data=$(PostFTLData "domains:batchDelete" "${json}") + + # If there is an .error object in the returned data, display it + local error + error=$(jq --compact-output <<< "${data}" '.error') + if [[ $error != "null" && $error != "" ]]; then + echo -e " ${CROSS} Failed to remove domain(s):" + echo -e " $(jq <<< "${data}" '.error')" + elif [[ "${verbose}" == true ]]; then + echo -e " ${TICK} Removed ${#domList[@]} domain(s):" + # Loop through the domains and display them + for dom in "${domList[@]}"; do + echo -e " - ${COL_BLUE}${dom}${COL_NC}" + done fi - # Domain found in the table, remove it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Removing ${domain} from the ${requestedListname}..." - fi - reload=true - # Remove it from the current list - pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" + # Log out + LogoutAPI } Displaylist() { - local count num_pipes domain enabled status nicedate requestedListname + local data - requestedListname="$(GetListnameFromTypeId "${typeId}")" - data="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" - - if [[ -z $data ]]; then - echo -e "Not showing empty list" - else - echo -e "Displaying ${requestedListname}:" - count=1 - while IFS= read -r line - do - # Count number of pipes seen in this line - # This is necessary because we can only detect the pipe separating the fields - # from the end backwards as the domain (which is the first field) may contain - # pipe symbols as they are perfectly valid regex filter control characters - num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")" - - # Extract domain and enabled status based on the obtained number of pipe characters - domain="$(cut -d'|' -f"-$((num_pipes-1))" <<< "${line}")" - enabled="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")" - datemod="$(cut -d'|' -f"$((num_pipes+1))" <<< "${line}")" - - # Translate boolean status into human readable string - if [[ "${enabled}" -eq 1 ]]; then - status="enabled" - else - status="disabled" - fi - - # Get nice representation of numerical date stored in database - nicedate=$(date --rfc-2822 -d "@${datemod}") - - echo " ${count}: ${domain} (${status}, last modified ${nicedate})" - count=$((count+1)) - done <<< "${data}" + # if either typeId or kindId is empty, we cannot display the list + if [[ -z "${typeId}" ]] || [[ -z "${kindId}" ]]; then + echo " ${CROSS} Unable to display list. Please specify a list type and kind." + exit 1 fi - exit 0; -} -NukeList() { - count=$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") - listname="$(GetListnameFromTypeId "${typeId}")" - if [ "$count" -gt 0 ];then - pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" - echo " ${TICK} Removed ${count} domain(s) from the ${listname}" + # Authenticate with the API + LoginAPI + + # Send the request + data=$(GetFTLData "domains/${typeId}/${kindId}") + + # Display the list + num=$(echo "${data}" | jq '.domains | length') + if [[ "${num}" -gt 0 ]]; then + echo -e " ${TICK} Found ${num} domain(s) in the ${kindId} ${typeId}list:" + for i in $(seq 0 $((num-1))); do + echo -e " - ${COL_BLUE}$(echo "${data}" | jq --compact-output ".domains[$i].domain")${COL_NC}" + echo -e " Comment: $(echo "${data}" | jq --compact-output ".domains[$i].comment")" + echo -e " Groups: $(echo "${data}" | jq --compact-output ".domains[$i].groups")" + echo -e " Added: $(date -d @"$(echo "${data}" | jq --compact-output ".domains[$i].date_added")")" + echo -e " Last modified: $(date -d @"$(echo "${data}" | jq --compact-output ".domains[$i].date_modified")")" + done else - echo " ${INFO} ${listname} already empty. Nothing to do!" + echo -e " ${INFO} No domains found in the ${kindId} ${typeId}list" fi - exit 0; + + # Log out + LogoutAPI + + # Return early without adding/deleting domains + exit 0 } GetComment() { @@ -272,38 +198,30 @@ GetComment() { while (( "$#" )); do case "${1}" in - "-w" | "whitelist" ) typeId=0;; - "-b" | "blacklist" ) typeId=1;; - "--white-regex" | "white-regex" ) typeId=2;; - "--white-wild" | "white-wild" ) typeId=2; wildcard=true;; - "--wild" | "wildcard" ) typeId=3; wildcard=true;; - "--regex" | "regex" ) typeId=3;; - "-nr"| "--noreload" ) noReloadRequested=true;; + "-a" | "allowlist" ) kindId="exact"; typeId="allow"; abbrv="-a";; + "-b" | "denylist" ) kindId="exact"; typeId="deny"; abbrv="-b";; + "--allow-regex" | "allow-regex" ) kindId="regex"; typeId="allow"; abbrv="--allow-regex";; + "--allow-wild" | "allow-wild" ) kindId="regex"; typeId="allow"; wildcard=true; abbrv="--allow-wild";; + "--regex" | "regex" ) kindId="regex"; typeId="deny"; abbrv="--regex";; + "--wild" | "wildcard" ) kindId="regex"; typeId="deny"; wildcard=true; abbrv="--wild";; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; "-h" | "--help" ) helpFunc;; "-l" | "--list" ) Displaylist;; - "--nuke" ) NukeList;; - "--web" ) web=true;; "--comment" ) GetComment "${2}"; shift;; - * ) ValidateDomain "${1}";; + * ) CreateDomainList "${1}";; esac shift done shift -if [[ ${domaincount} == 0 ]]; then +if [[ ${#domList[@]} == 0 ]]; then helpFunc fi -ProcessDomainList - -# Used on web interface -if $web; then - echo "DONE" -fi - -if [[ ${reload} == true && ${noReloadRequested} == false ]]; then - pihole restartdns reload-lists +if ${addmode}; then + AddDomain +else + RemoveDomain fi diff --git a/pihole b/pihole index f7963d73..ce46fd0f 100755 --- a/pihole +++ b/pihole @@ -537,12 +537,12 @@ case "${1}" in "tricorder" ) tricorderFunc;; # we need to add all arguments that require sudo power to not trigger the * argument - "-w" | "whitelist" ) ;; - "-b" | "blacklist" ) ;; - "--wild" | "wildcard" ) ;; - "--regex" | "regex" ) ;; - "--white-regex" | "white-regex" ) ;; - "--white-wild" | "white-wild" ) ;; + "-a" | "allowlist" ) need_root=0;; + "-b" | "blocklist" | "denylist" ) need_root=0;; + "--wild" | "wildcard" ) need_root=0;; + "--regex" | "regex" ) need_root=0;; + "--allow-regex" | "allow-regex" ) need_root=0;; + "--allow-wild" | "allow-wild" ) need_root=0;; "-f" | "flush" ) ;; "-up" | "updatePihole" ) ;; "-r" | "reconfigure" ) ;; @@ -592,12 +592,12 @@ fi # Handle redirecting to specific functions based on arguments case "${1}" in - "-w" | "whitelist" ) listFunc "$@";; - "-b" | "blacklist" ) listFunc "$@";; + "-a" | "allowlist" ) listFunc "$@";; + "-b" | "blocklist" | "denylist" ) listFunc "$@";; "--wild" | "wildcard" ) listFunc "$@";; "--regex" | "regex" ) listFunc "$@";; - "--white-regex" | "white-regex" ) listFunc "$@";; - "--white-wild" | "white-wild" ) listFunc "$@";; + "--allow-regex" | "allow-regex" ) listFunc "$@";; + "--allow-wild" | "allow-wild" ) listFunc "$@";; "-d" | "debug" ) debugFunc "$@";; "-f" | "flush" ) flushFunc "$@";; "-up" | "updatePihole" ) updatePiholeFunc "$@";;