From a6565bf9a17ad4998b00ca239be9044be7f51674 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 23 Nov 2023 22:07:31 +0100 Subject: [PATCH 1/5] Support special webserver.port ports ending in "s" (secure) and "r" (redirect) Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 43 ++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 449f146f..000c0717 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -21,14 +21,31 @@ TestAPIAvailability() { # as we are running locally, we can get the port value from FTL directly - PORT="$(pihole-FTL --config webserver.port)" - PORT="${PORT%%,*}" + local ports port availabilityResonse + ports="$(pihole-FTL --config webserver.port)" + port="${ports%%,*}" - availabilityResonse=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${PORT}/api/auth") + # if the port ends with an "s", it is a secure connection + if [ "${port#"${port%?}"}" = "s" ]; then + # remove the "s" from the port + API_PROT="https" + API_PORT="${port%?}" + elif [ "${port#"${port%?}"}" = "r" ]; then + # if the port ends in "r", it is a redirect + API_PROT="http" + # remove the "r" from the port + API_PORT="${port%?}" + else + API_PROT="http" + API_PORT="${port}" + fi - # test if http status code was 200 (OK) or 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then - echo "API not available at: http://localhost:${PORT}/api" + API_URL="${API_PROT}://localhost:${API_PORT}/api" + availabilityResonse=$(curl -skSL -o /dev/null -w "%{http_code}" "${API_URL}/auth") + + # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + echo "API not available at: ${API_URL}" echo "Exiting." exit 1 fi @@ -54,15 +71,15 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl --silent -X POST "http://localhost:${PORT}/api/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skSL -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" exit 1 fi - # obtain validity and session ID from session response - validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) - SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) + # obtain validity and session ID from session response + validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) + SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) } DeleteSession() { @@ -70,7 +87,7 @@ DeleteSession() { # SID is not null (successful authenthication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "http://localhost:${PORT}/api/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skSL -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; @@ -84,14 +101,14 @@ DeleteSession() { GetFTLData() { local data response status # get the data from querying the API as well as the http status code - response=$(curl -s -w "%{http_code}" -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) + response=$(curl -skSL -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) # status are the last 3 characters status=$(printf %s "${response#"${response%???}"}") # data is everything from response without the last 3 characters data=$(printf %s "${response%???}") - if [ "${status}" = 200 ]; then + if [ "${status}" = 200 ] || [ "${status}" = 308 ]; then # response OK echo "${data}" elif [ "${status}" = 000 ]; then From 974fea592df99d97332763b6f9cf9812a6c907a4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 00:05:55 +0100 Subject: [PATCH 2/5] Iterate over ports, skip redirected ports Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 64 +++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 000c0717..b50e416a 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -25,26 +25,48 @@ TestAPIAvailability() { ports="$(pihole-FTL --config webserver.port)" port="${ports%%,*}" - # if the port ends with an "s", it is a secure connection - if [ "${port#"${port%?}"}" = "s" ]; then - # remove the "s" from the port - API_PROT="https" - API_PORT="${port%?}" - elif [ "${port#"${port%?}"}" = "r" ]; then - # if the port ends in "r", it is a redirect - API_PROT="http" - # remove the "r" from the port - API_PORT="${port%?}" - else - API_PROT="http" - API_PORT="${port}" - fi + # Iterate over comma separated list of ports + while [ "${port}" != "${ports}" ]; do + # if the port ends with an "s", it is a secure connection + if [ "${port#"${port%?}"}" = "s" ]; then + # remove the "s" from the port + API_PROT="https" + API_PORT="${port%?}" + elif [ "${port#"${port%?}"}" = "r" ]; then + # Ignore this port + API_PORT="0" + else + API_PROT="http" + API_PORT="${port}" + fi - API_URL="${API_PROT}://localhost:${API_PORT}/api" - availabilityResonse=$(curl -skSL -o /dev/null -w "%{http_code}" "${API_URL}/auth") + if [ ! "${API_PORT}" = "0" ]; then + # If the port is of form "ip:port", we need to remove everything before + # the last ":" in the string, e.g., "[::]:80" -> "80" + if [ "${API_PORT#*:}" != "${API_PORT}" ]; then + API_PORT="${API_PORT##*:}" + fi - # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + API_URL="${API_PROT}://localhost:${API_PORT}/api" + availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") + + # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + API_PORT="0" + else + # API is available at this port/protocol combination + break + fi + fi + + # remove the first port from the list + ports="${ports#*,}" + # get the next port + port="${ports%%,*}" + done + + # if API_PORT is 0, no working API port was found + if [ "${API_PORT}" = "0" ]; then echo "API not available at: ${API_URL}" echo "Exiting." exit 1 @@ -71,7 +93,7 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl -skSL -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skS -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" @@ -87,7 +109,7 @@ DeleteSession() { # SID is not null (successful authenthication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -skSL -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; @@ -101,7 +123,7 @@ DeleteSession() { GetFTLData() { local data response status # get the data from querying the API as well as the http status code - response=$(curl -skSL -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) + response=$(curl -skS -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) # status are the last 3 characters status=$(printf %s "${response#"${response%???}"}") From 6016131280ce5f24cc53fa7989026f593f4d2d4f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 22:59:49 +0100 Subject: [PATCH 3/5] Ensure we also check the last port Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index b50e416a..2952fb43 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -26,16 +26,18 @@ TestAPIAvailability() { port="${ports%%,*}" # Iterate over comma separated list of ports - while [ "${port}" != "${ports}" ]; do + while [ -n "${ports}" ]; do # if the port ends with an "s", it is a secure connection if [ "${port#"${port%?}"}" = "s" ]; then # remove the "s" from the port API_PROT="https" API_PORT="${port%?}" elif [ "${port#"${port%?}"}" = "r" ]; then - # Ignore this port + # Ignore this port, the client may not be able to follow the + # redirected target when FTL is not used as local resolver API_PORT="0" else + # otherwise it is an insecure (plain HTTP) connection API_PROT="http" API_PORT="${port}" fi @@ -50,8 +52,9 @@ TestAPIAvailability() { API_URL="${API_PROT}://localhost:${API_PORT}/api" availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") - # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # API is not available at this port/protocol combination API_PORT="0" else # API is available at this port/protocol combination @@ -59,9 +62,9 @@ TestAPIAvailability() { fi fi - # remove the first port from the list + # If the loop has not been broken, remove the first port from the list + # and get the next port ports="${ports#*,}" - # get the next port port="${ports%%,*}" done @@ -192,3 +195,6 @@ secretRead() { # restore original terminal settings stty "${stty_orig}" } + + +TestAPIAvailability From 96bf07863f6bdd4d9cad91d5444fef5f85d47b61 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 09:10:06 +0100 Subject: [PATCH 4/5] Use CHAOS TXT local.api.txt instead of trying to parse pihole-FTL --config webserver.ports Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 90 +++++++++++++++++++-------------------- advanced/Scripts/query.sh | 6 +-- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 2952fb43..46da37cd 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -21,62 +21,60 @@ TestAPIAvailability() { # as we are running locally, we can get the port value from FTL directly - local ports port availabilityResonse - ports="$(pihole-FTL --config webserver.port)" - port="${ports%%,*}" + local chaos_api_list availabilityResonse - # Iterate over comma separated list of ports - while [ -n "${ports}" ]; do - # if the port ends with an "s", it is a secure connection - if [ "${port#"${port%?}"}" = "s" ]; then - # remove the "s" from the port - API_PROT="https" - API_PORT="${port%?}" - elif [ "${port#"${port%?}"}" = "r" ]; then - # Ignore this port, the client may not be able to follow the - # redirected target when FTL is not used as local resolver - API_PORT="0" + # Query the API URLs from FTL using CHAOS TXT local.api.ftl + # The result is a space-separated enumeration of full URLs + # e.g., "http://localhost:80/api" "https://localhost:443/api" + chaos_api_list="$(dig +short chaos txt local.api.ftl @127.0.0.1)" + + # If the query was not successful, the variable is empty + if [ -z "${chaos_api_list}" ]; then + echo "API not available. Please check connectivity" + exit 1 + fi + + # Iterate over space-separated list of URLs + while [ -n "${chaos_api_list}" ]; do + # Get the first URL + API_URL="${chaos_api_list%% *}" + # Strip leading and trailing quotes + API_URL="${API_URL%\"}" + API_URL="${API_URL#\"}" + + # Test if the API is available at this URL + availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") + + # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # API is not available at this port/protocol combination + API_PORT="" else - # otherwise it is an insecure (plain HTTP) connection - API_PROT="http" - API_PORT="${port}" + # API is available at this URL combination + break fi - if [ ! "${API_PORT}" = "0" ]; then - # If the port is of form "ip:port", we need to remove everything before - # the last ":" in the string, e.g., "[::]:80" -> "80" - if [ "${API_PORT#*:}" != "${API_PORT}" ]; then - API_PORT="${API_PORT##*:}" - fi + # Remove the first URL from the list + local last_api_list + last_api_list="${chaos_api_list}" + chaos_api_list="${chaos_api_list#* }" - API_URL="${API_PROT}://localhost:${API_PORT}/api" - availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") - - # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then - # API is not available at this port/protocol combination - API_PORT="0" - else - # API is available at this port/protocol combination - break - fi + # If the list did not change, we are at the last element + if [ "${last_api_list}" = "${chaos_api_list}" ]; then + # Remove the last element + chaos_api_list="" fi - - # If the loop has not been broken, remove the first port from the list - # and get the next port - ports="${ports#*,}" - port="${ports%%,*}" done - # if API_PORT is 0, no working API port was found - if [ "${API_PORT}" = "0" ]; then + # if API_PORT is empty, no working API port was found + if [ -n "${API_PORT}" ]; then echo "API not available at: ${API_URL}" echo "Exiting." exit 1 fi } -Authenthication() { +Authentication() { # Try to authenticate LoginAPI @@ -96,7 +94,7 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl -skS -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skS -X POST "${API_URL}auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" @@ -108,11 +106,11 @@ LoginAPI() { } DeleteSession() { - # if a valid Session exists (no password required or successful authenthication) and - # SID is not null (successful authenthication only), delete the session + # if a valid Session exists (no password required or successful Authentication) and + # SID is not null (successful Authentication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 2279df85..62d29d5b 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -121,14 +121,14 @@ Main(){ # or b) for the /search endpoint (webserver.api.searchAPIauth) no authentication is required. # Therefore, we try to query directly without authentication but do authenticat if 401 is returned - data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") if [ "${data}" = 401 ]; then # Unauthenticated, so authenticate with the FTL server required - Authenthication + Authentication # send query again - data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") fi GenerateOutput "${data}" From 32a741b5c74bab31ed5c02f5ba33f65f631fea7f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 2 Dec 2023 22:42:36 +0100 Subject: [PATCH 5/5] We do not follow 308 but FTL also doesn't suggest it Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 46da37cd..b7bc2a86 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -25,7 +25,7 @@ TestAPIAvailability() { # Query the API URLs from FTL using CHAOS TXT local.api.ftl # The result is a space-separated enumeration of full URLs - # e.g., "http://localhost:80/api" "https://localhost:443/api" + # e.g., "http://localhost:80/api/" "https://localhost:443/api/" chaos_api_list="$(dig +short chaos txt local.api.ftl @127.0.0.1)" # If the query was not successful, the variable is empty @@ -45,8 +45,8 @@ TestAPIAvailability() { # Test if the API is available at this URL availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") - # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # Test if http status code was 200 (OK) or 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then # API is not available at this port/protocol combination API_PORT="" else @@ -131,7 +131,7 @@ GetFTLData() { # data is everything from response without the last 3 characters data=$(printf %s "${response%???}") - if [ "${status}" = 200 ] || [ "${status}" = 308 ]; then + if [ "${status}" = 200 ]; then # response OK echo "${data}" elif [ "${status}" = 000 ]; then @@ -193,6 +193,3 @@ secretRead() { # restore original terminal settings stty "${stty_orig}" } - - -TestAPIAvailability