From 03d93aa19a61cbe3e23dea47c3bdd9cd2c66287a Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 20 May 2019 20:58:57 -0700 Subject: [PATCH 1/7] Update debug script with gravity DB changes Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 67 +++++++++++++++++---------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 1010f26c..3a5c482f 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -89,16 +89,12 @@ PIHOLE_WILDCARD_CONFIG_FILE="${DNSMASQ_D_DIRECTORY}/03-wildcard.conf" WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" #WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" -PIHOLE_DEFAULT_AD_LISTS="${PIHOLE_DIRECTORY}/adlists.default" -PIHOLE_USER_DEFINED_AD_LISTS="${PIHOLE_DIRECTORY}/adlists.list" -PIHOLE_BLACKLIST_FILE="${PIHOLE_DIRECTORY}/blacklist.txt" -PIHOLE_BLOCKLIST_FILE="${PIHOLE_DIRECTORY}/gravity.list" PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_SETUP_VARS_FILE="${PIHOLE_DIRECTORY}/setupVars.conf" -PIHOLE_WHITELIST_FILE="${PIHOLE_DIRECTORY}/whitelist.txt" +PIHOLE_GRAVITY_DB_FILE="${PIHOLE_DIRECTORY}/gravity.db" PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole" PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE" @@ -142,16 +138,11 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}" "${PIHOLE_DHCP_CONFIG_FILE}" "${PIHOLE_WILDCARD_CONFIG_FILE}" "${WEB_SERVER_CONFIG_FILE}" -"${PIHOLE_DEFAULT_AD_LISTS}" -"${PIHOLE_USER_DEFINED_AD_LISTS}" -"${PIHOLE_BLACKLIST_FILE}" -"${PIHOLE_BLOCKLIST_FILE}" "${PIHOLE_INSTALL_LOG_FILE}" "${PIHOLE_RAW_BLOCKLIST_FILES}" "${PIHOLE_LOCAL_HOSTS_FILE}" "${PIHOLE_LOGROTATE_FILE}" "${PIHOLE_SETUP_VARS_FILE}" -"${PIHOLE_WHITELIST_FILE}" "${PIHOLE_COMMAND}" "${PIHOLE_COLTABLE_FILE}" "${FTL_PID}" @@ -793,7 +784,7 @@ dig_at() { # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(shuf -n 1 "${PIHOLE_BLOCKLIST_FILE}") + random_url=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1") # First, do a dig on localhost to see if Pi-hole can use itself to block a domain if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then @@ -975,8 +966,7 @@ list_files_in_dir() { if [[ -d "${dir_to_parse}/${each_file}" ]]; then # If it's a directoy, do nothing : - elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_BLOCKLIST_FILE}" ]] || \ - [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \ + elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_RAW_BLOCKLIST_FILES}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_INSTALL_LOG_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_SETUP_VARS_FILE}" ]] || \ @@ -1061,31 +1051,43 @@ head_tail_log() { IFS="$OLD_IFS" } -analyze_gravity_list() { - echo_current_diagnostic "Gravity list" - local head_line - local tail_line - # Put the current Internal Field Separator into another variable so it can be restored later +show_adlists() { + echo_current_diagnostic "Adlists" + OLD_IFS="$IFS" - # Get the lines that are in the file(s) and store them in an array for parsing later IFS=$'\r\n' + local adlists=() + mapfile -t adlists < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT address FROM vw_adlists") + + for line in "${adlists[@]}"; do + log_write " ${line}" + done + + IFS="$OLD_IFS" +} + +analyze_gravity_list() { + echo_current_diagnostic "Gravity List and Database" + local gravity_permissions - gravity_permissions=$(ls -ld "${PIHOLE_BLOCKLIST_FILE}") + gravity_permissions=$(ls -ld "${PIHOLE_GRAVITY_DB_FILE}") log_write "${COL_GREEN}${gravity_permissions}${COL_NC}" - local gravity_head=() - mapfile -t gravity_head < <(head -n 4 ${PIHOLE_BLOCKLIST_FILE}) - log_write " ${COL_CYAN}-----head of $(basename ${PIHOLE_BLOCKLIST_FILE})------${COL_NC}" - for head_line in "${gravity_head[@]}"; do - log_write " ${head_line}" + + local gravity_size + gravity_size=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT COUNT(*) FROM vw_gravity") + log_write " Size: ${COL_CYAN}${gravity_size}${COL_NC} entries" + + OLD_IFS="$IFS" + IFS=$'\r\n' + local gravity_sample=() + mapfile -t gravity_sample < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") + log_write " ${COL_CYAN}----- First 10 Domains -----${COL_NC}" + + for line in "${gravity_sample[@]}"; do + log_write " ${line}" done + log_write "" - local gravity_tail=() - mapfile -t gravity_tail < <(tail -n 4 ${PIHOLE_BLOCKLIST_FILE}) - log_write " ${COL_CYAN}-----tail of $(basename ${PIHOLE_BLOCKLIST_FILE})------${COL_NC}" - for tail_line in "${gravity_tail[@]}"; do - log_write " ${tail_line}" - done - # Set the IFS back to what it was IFS="$OLD_IFS" } @@ -1236,6 +1238,7 @@ process_status parse_setup_vars check_x_headers analyze_gravity_list +show_adlists show_content_of_pihole_files parse_locale analyze_pihole_log From 3f05efd60f4e4ff5630621d3a1d50fd81e1e1807 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 20 May 2019 21:02:31 -0700 Subject: [PATCH 2/7] Add extra newline Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 3a5c482f..82660d61 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1076,6 +1076,7 @@ analyze_gravity_list() { local gravity_size gravity_size=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT COUNT(*) FROM vw_gravity") log_write " Size: ${COL_CYAN}${gravity_size}${COL_NC} entries" + log_write "" OLD_IFS="$IFS" IFS=$'\r\n' From 807ce0af4ed7d59830c1ae26eaffb6757c320699 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 20 May 2019 21:15:22 -0700 Subject: [PATCH 3/7] Show whitelist, blacklist, and regexlist details Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 82660d61..c491b1c6 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1051,21 +1051,40 @@ head_tail_log() { IFS="$OLD_IFS" } -show_adlists() { - echo_current_diagnostic "Adlists" +show_db_entries() { + local title="${1}" + local query="${2}" + + echo_current_diagnostic "${title}" OLD_IFS="$IFS" IFS=$'\r\n' - local adlists=() - mapfile -t adlists < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT address FROM vw_adlists") + local entries=() + mapfile -t entries < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" -cmd ".headers on" "${query}") - for line in "${adlists[@]}"; do + for line in "${entries[@]}"; do log_write " ${line}" done IFS="$OLD_IFS" } +show_adlists() { + show_db_entries "Adlists" "SELECT * FROM adlists" +} + +show_whitelist() { + show_db_entries "Whitelist" "SELECT * FROM whitelist" +} + +show_blacklist() { + show_db_entries "Blacklist" "SELECT * FROM blacklist" +} + +show_regexlist() { + show_db_entries "Regexlist" "SELECT * FROM regex" +} + analyze_gravity_list() { echo_current_diagnostic "Gravity List and Database" @@ -1240,6 +1259,9 @@ parse_setup_vars check_x_headers analyze_gravity_list show_adlists +show_whitelist +show_blacklist +show_regexlist show_content_of_pihole_files parse_locale analyze_pihole_log From 7b5fc60e003263b58fc7f1ebf8ce3bc3f6b74796 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 20 May 2019 21:20:38 -0700 Subject: [PATCH 4/7] Improve table formatting Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c491b1c6..e56d1f94 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1060,7 +1060,7 @@ show_db_entries() { OLD_IFS="$IFS" IFS=$'\r\n' local entries=() - mapfile -t entries < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" -cmd ".headers on" "${query}") + mapfile -t entries < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" -cmd ".headers on" -cmd ".mode column" "${query}") for line in "${entries[@]}"; do log_write " ${line}" From a3e1473ac10411e2fc72952bb2fa7983393b3b17 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 20 May 2019 21:33:09 -0700 Subject: [PATCH 5/7] Set explicit column widths to prevent text from getting cut off Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index e56d1f94..816652e0 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1054,13 +1054,20 @@ head_tail_log() { show_db_entries() { local title="${1}" local query="${2}" + local widths="${3}" echo_current_diagnostic "${title}" OLD_IFS="$IFS" IFS=$'\r\n' local entries=() - mapfile -t entries < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" -cmd ".headers on" -cmd ".mode column" "${query}") + mapfile -t entries < <(\ + sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \ + -cmd ".headers on" \ + -cmd ".mode column" \ + -cmd ".width ${widths}" \ + "${query}"\ + ) for line in "${entries[@]}"; do log_write " ${line}" @@ -1070,19 +1077,19 @@ show_db_entries() { } show_adlists() { - show_db_entries "Adlists" "SELECT * FROM adlists" + show_db_entries "Adlists" "SELECT * FROM adlists" "2 100 7 10 13 50" } show_whitelist() { - show_db_entries "Whitelist" "SELECT * FROM whitelist" + show_db_entries "Whitelist" "SELECT * FROM whitelist" "2 100 7 10 13 50" } show_blacklist() { - show_db_entries "Blacklist" "SELECT * FROM blacklist" + show_db_entries "Blacklist" "SELECT * FROM blacklist" "2 100 7 10 13 50" } show_regexlist() { - show_db_entries "Regexlist" "SELECT * FROM regex" + show_db_entries "Regexlist" "SELECT * FROM regex" "2 100 7 10 13 50" } analyze_gravity_list() { From 5796054305e4b6842d95f8f1259df35862a34ba4 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 20 May 2019 21:59:18 -0700 Subject: [PATCH 6/7] Increase ID column width to 4 Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 816652e0..07a11ff2 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1077,19 +1077,19 @@ show_db_entries() { } show_adlists() { - show_db_entries "Adlists" "SELECT * FROM adlists" "2 100 7 10 13 50" + show_db_entries "Adlists" "SELECT * FROM adlists" "4 100 7 10 13 50" } show_whitelist() { - show_db_entries "Whitelist" "SELECT * FROM whitelist" "2 100 7 10 13 50" + show_db_entries "Whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50" } show_blacklist() { - show_db_entries "Blacklist" "SELECT * FROM blacklist" "2 100 7 10 13 50" + show_db_entries "Blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50" } show_regexlist() { - show_db_entries "Regexlist" "SELECT * FROM regex" "2 100 7 10 13 50" + show_db_entries "Regexlist" "SELECT * FROM regex" "4 100 7 10 13 50" } analyze_gravity_list() { From bfb99c361c6db92a728760ba34edb63ffca6006c Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Tue, 21 May 2019 17:12:47 -0700 Subject: [PATCH 7/7] Note that the gravity size does not include the blacklist entries Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 07a11ff2..b31bbdc5 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1101,7 +1101,7 @@ analyze_gravity_list() { local gravity_size gravity_size=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT COUNT(*) FROM vw_gravity") - log_write " Size: ${COL_CYAN}${gravity_size}${COL_NC} entries" + log_write " Size (excluding blacklist): ${COL_CYAN}${gravity_size}${COL_NC} entries" log_write "" OLD_IFS="$IFS"