From 4fd0f15d903472e2e56ee3604a16d48d982ab01b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 16 Feb 2023 15:21:18 +0100 Subject: [PATCH 001/462] Ignore commented lines when reding PRIVACYLEVEL from config file Create dedicated getVal function in utils.sh as it might be useful somewhere else Account for tailing comments and $key not being on the first line MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/utils.sh | 22 ++++++++++++++++++++-- automated install/basic-install.sh | 3 ++- test/test_any_utils.py | 16 ++++++++++++++++ 3 files changed, 38 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 37516472..a1178265 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -44,7 +44,7 @@ addOrEditKeyValPair() { } ####################### -# Takes two arguments: file, and key. +# Takes two arguments: file and key. # Adds a key to target file # # Example usage: @@ -64,7 +64,7 @@ addKey(){ } ####################### -# Takes two arguments: file, and key. +# Takes two arguments: file and key. # Deletes a key or key/value pair from target file # # Example usage: @@ -76,6 +76,24 @@ removeKey() { sed -i "/^${key}/d" "${file}" } +####################### +# Takes two arguments: file and key. +# Returns the value of a given key from target file +# - ignores all commented lines +# - only returns the first value if multiple identical keys exist +# +# +# Example usage: +# getVal "/etc/pihole/setupVars.conf" "PIHOLE_DNS_1" +####################### +getVal() { + local file="${1}" + local key="${2}" + local value + value=$(sed -e '/^[[:blank:]]*#/d' "${file}" | grep "${key}" | awk -F "=" 'NR==1{printf$2}') + printf "%s" "$value" +} + ####################### # returns FTL's current telnet API port based on the setting in /etc/pihole-FTL.conf diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 05bc0e4e..a9398d90 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2612,7 +2612,8 @@ main() { # Get the privacy level if it exists (default is 0) if [[ -f "${FTL_CONFIG_FILE}" ]]; then - PRIVACY_LEVEL=$(sed -ne 's/PRIVACYLEVEL=\(.*\)/\1/p' "${FTL_CONFIG_FILE}") + # use getVal from utils.sh to get PRIVACYLEVEL + PRIVACY_LEVEL=$(getVal "${FTL_CONFIG_FILE}" "PRIVACYLEVEL") # If no setting was found, default to 0 PRIVACY_LEVEL="${PRIVACY_LEVEL:-0}" diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 5b4075d9..6c920161 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -62,6 +62,22 @@ def test_key_removal_works(host): assert expected_stdout == output.stdout +def test_get_value_works(host): + """Confirms getVal returns the correct value for a given key""" + output = host.run( + """ + source /opt/pihole/utils.sh + echo "Somekey=xxx" >> /tmp/testfile + echo "#Testkey=1234" >> /tmp/testfile + echo "Testkey=5678" >> /tmp/testfile + echo "Testkey=abcd" >> /tmp/testfile + getVal "/tmp/testfile" "Testkey" + """ + ) + expected_stdout = "5678" + assert expected_stdout == output.stdout + + def test_getFTLAPIPort_default(host): """Confirms getFTLAPIPort returns the default API port""" output = host.run( From ca74152d1d5a5fc179314458a0a96a1843156f0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 16 Feb 2023 23:11:00 +0100 Subject: [PATCH 002/462] Allow adding ABP style blocklists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adam Warner Signed-off-by: Christian König --- gravity.sh | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/gravity.sh b/gravity.sh index 28c28a9e..d3bb4a07 100755 --- a/gravity.sh +++ b/gravity.sh @@ -519,25 +519,31 @@ gravity_DownloadBlocklists() { gravity_Blackbody=true } -# num_total_imported_domains increases for each list processed -num_total_imported_domains=0 -num_domains=0 -num_non_domains=0 parseList() { - local adlistID="${1}" src="${2}" target="${3}" non_domains sample_non_domains - # This sed does the following things: - # 1. Remove all lines containing no domains - # 2. Remove all domains containing invalid characters. Valid are: a-z, A-Z, 0-9, dot (.), minus (-), underscore (_) - # 3. Append ,adlistID to every line - # 4. Remove trailing period (see https://github.com/pi-hole/pi-hole/issues/4701) - # 5. Ensures there is a newline on the last line - sed -r "/([^\.]+\.)+[^\.]{2,}/!d;/[^a-zA-Z0-9.\_-]/d;s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${src}" >> "${target}" + local adlistID="${1}" src="${2}" target="${3}" temp_file non_domains sample_non_domains - # Find lines containing no domains or with invalid characters (see above) + # Create a temporary file for the sed magic instead of using "${target}" directly + # this allows to split the sed commands to improve readability + temp_file="$(mktemp -p "/tmp" --suffix=".gravity")" + + # 1. Add all valid domains (adapted from https://stackoverflow.com/a/30007882) + # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already + sed -r "/^([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/!d" "${src}" > "${temp_file}" + # 2. Add all supported ABP style lines (||subdomain.domain.tlp^) + sed -r "/^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\^$/!d" "${src}" >> "${temp_file}" + + # Find lines containing no domains or with invalid characters (not matching regex above) + # This is simply everything that is not in $temp_file compared to $src # Remove duplicates from the list - mapfile -t non_domains <<< "$(sed -r "/([^\.]+\.)+[^\.]{2,}/d" < "${src}")" - mapfile -t -O "${#non_domains[@]}" non_domains <<< "$(sed -r "/[^a-zA-Z0-9.\_-]/!d" < "${src}")" - IFS=" " read -r -a non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | sort -u | tr '\n' ' ')" + mapfile -t non_domains < <(grep -Fvf "${temp_file}" "${src}" | sort -u ) + + # 3. Remove trailing period (see https://github.com/pi-hole/pi-hole/issues/4701) + # 4. Append ,adlistID to every line + # 5. Ensures there is a newline on the last line + sed -i "s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${temp_file}" + + # concatenate the temporary file to the target file + cat "${temp_file}" >> "${target}" # A list of items of common local hostnames not to report as unusable # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files @@ -553,13 +559,8 @@ parseList() { # Get a sample of non-domain entries, limited to 5 (the list should already have been de-duplicated) IFS=" " read -r -a sample_non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | head -n 5 | tr '\n' ' ')" - local tmp_new_imported_total - # Get the new number of domains in destination file - tmp_new_imported_total="$(grep -c "^" "${target}")" - # Number of imported lines for this file is the difference between the new total and the old total. (Or, the number of domains we just added.) - num_domains="$(( tmp_new_imported_total-num_total_imported_domains ))" - # Replace the running total with the new total. - num_total_imported_domains="$tmp_new_imported_total" + # Get the number of domains added + num_domains="$(grep -c "^" "${temp_file}")" # Get the number of non_domains (this is the number of entries left after stripping the source of comments/duplicates/false positives/domains) num_non_domains="${#non_domains[@]}" From 1f3f8491065949f38faac75879764f7edf9ce94e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 19 Feb 2023 17:47:10 +0000 Subject: [PATCH 003/462] Remove lines starting with ! or [ to account for ABP style comments and header Also splits the piped "one-liner" in ParseFileIntoDomains into individually commented commands (makes for easier reading and debugging) Signed-off-by: Adam Warner --- gravity.sh | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/gravity.sh b/gravity.sh index d3bb4a07..fb7f42f1 100755 --- a/gravity.sh +++ b/gravity.sh @@ -745,18 +745,30 @@ gravity_ParseFileIntoDomains() { # Most of the lists downloaded are already in hosts file format but the spacing/formatting is not contiguous # This helps with that and makes it easier to read # It also helps with debugging so each stage of the script can be researched more in depth - # 1) Remove carriage returns - # 2) Convert all characters to lowercase - # 3) Remove comments (text starting with "#", include possible spaces before the hash sign) + # 1) Convert all characters to lowercase + tr '[:upper:]' '[:lower:]' < "${src}" > "${destination}" + + # 2) Remove carriage returns + sed -i 's/\r$//' "${destination}" + + # 3a) Remove comments (text starting with "#", include possible spaces before the hash sign) + sed -i 's/\s*#.*//g' "${destination}" + + # 3b) Remove lines starting with ! (ABP Comments) + sed -i 's/\s*!.*//g' "${destination}" + + # 3c) Remove lines starting with [ (ABP Header) + sed -i 's/\s*\[.*//g' "${destination}" + # 4) Remove lines containing "/" - # 5) Remove leading tabs, spaces, etc. + sed -i -r '/(\/).*$/d' "${destination}" + + # 5) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) + sed -i -r 's/^.*\s+//g' "${destination}" + # 6) Remove empty lines - < "${src}" tr -d '\r' | \ - tr '[:upper:]' '[:lower:]' | \ - sed 's/\s*#.*//g' | \ - sed -r '/(\/).*$/d' | \ - sed -r 's/^.*\s+//g' | \ - sed '/^$/d'> "${destination}" + sed -i '/^$/d' "${destination}" + chmod 644 "${destination}" } From c5faf3d1744f100bbbfc800702c70ffc79eda02e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 19 Feb 2023 18:12:03 +0000 Subject: [PATCH 004/462] Use ',' as the separator char in query rather than the default '|' as we now expect some valid results to contain '|' Signed-off-by: Adam Warner --- advanced/Scripts/query.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index d48e9363..8717d328 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -116,7 +116,7 @@ scanDatabaseTable() { fi # Send prepared query to gravity database - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null + result="$(pihole-FTL sqlite3 -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null if [[ -z "${result}" ]]; then # Return early when there are no matches in this table return @@ -136,8 +136,8 @@ scanDatabaseTable() { # Loop over results and print them mapfile -t results <<< "${result}" for result in "${results[@]}"; do - domain="${result/|*}" - if [[ "${result#*|}" == "0" ]]; then + domain="${result/,*}" + if [[ "${result#*,}" == "0" ]]; then extra=" (disabled)" else extra="" @@ -212,10 +212,10 @@ if [[ -n "${exact}" ]]; then fi for result in "${results[@]}"; do - match="${result/|*/}" - extra="${result#*|}" - adlistAddress="${extra/|*/}" - extra="${extra#*|}" + match="${result/,*/}" + extra="${result#*,}" + adlistAddress="${extra/,*/}" + extra="${extra#*,}" if [[ "${extra}" == "0" ]]; then extra=" (disabled)" else From eedd93d7823ace35e2915e83a6dcc86b13abfd68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 22 Feb 2023 21:14:27 +0100 Subject: [PATCH 005/462] Remove temporary files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index fb7f42f1..57959b25 100755 --- a/gravity.sh +++ b/gravity.sh @@ -524,7 +524,13 @@ parseList() { # Create a temporary file for the sed magic instead of using "${target}" directly # this allows to split the sed commands to improve readability + # we use a file handle here and remove the temporary file immediately so the content will be deleted in any case + # when the script stops temp_file="$(mktemp -p "/tmp" --suffix=".gravity")" + exec 3>"$temp_file" + rm "${temp_file}" + temp_file="/proc/$$/fd/3" + # 1. Add all valid domains (adapted from https://stackoverflow.com/a/30007882) # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already @@ -540,10 +546,8 @@ parseList() { # 3. Remove trailing period (see https://github.com/pi-hole/pi-hole/issues/4701) # 4. Append ,adlistID to every line # 5. Ensures there is a newline on the last line - sed -i "s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${temp_file}" - - # concatenate the temporary file to the target file - cat "${temp_file}" >> "${target}" + # and write everything to the target file + sed "s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${temp_file}" >> "${target}" # A list of items of common local hostnames not to report as unusable # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files @@ -575,6 +579,9 @@ parseList() { else echo " ${INFO} Imported ${num_domains} domains" fi + + # close file handle + exec 3<&- } compareLists() { From 821c7dc190cdabc28f024e4688bff3670f028731 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 24 Feb 2023 22:18:41 +0100 Subject: [PATCH 006/462] Add info when list cotains ABP style domains MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 57959b25..451a40fc 100755 --- a/gravity.sh +++ b/gravity.sh @@ -520,23 +520,29 @@ gravity_DownloadBlocklists() { } parseList() { - local adlistID="${1}" src="${2}" target="${3}" temp_file non_domains sample_non_domains + local adlistID="${1}" src="${2}" target="${3}" temp_file temp_file_base non_domains sample_non_domains # Create a temporary file for the sed magic instead of using "${target}" directly # this allows to split the sed commands to improve readability # we use a file handle here and remove the temporary file immediately so the content will be deleted in any case # when the script stops - temp_file="$(mktemp -p "/tmp" --suffix=".gravity")" - exec 3>"$temp_file" - rm "${temp_file}" + temp_file_base="$(mktemp -p "/tmp" --suffix=".gravity")" + exec 3>"$temp_file_base" + rm "${temp_file_base}" temp_file="/proc/$$/fd/3" # 1. Add all valid domains (adapted from https://stackoverflow.com/a/30007882) # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already sed -r "/^([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/!d" "${src}" > "${temp_file}" - # 2. Add all supported ABP style lines (||subdomain.domain.tlp^) - sed -r "/^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\^$/!d" "${src}" >> "${temp_file}" + + # if there is at least one ABP style domains + if grep -E "^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" -m 1 -q "${src}"; then + echo " ${INFO} List contained AdBlock Plus style domains" + # 2. Add all supported ABP style lines (||subdomain.domain.tlp^) + sed -r "/^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\^$/!d" "${src}" >> "${temp_file}" + fi + # Find lines containing no domains or with invalid characters (not matching regex above) # This is simply everything that is not in $temp_file compared to $src From 6cb0be82caca4ca7b642a292f3d5a16ab1b47bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 26 Feb 2023 10:34:17 +0100 Subject: [PATCH 007/462] Add flag abp_domains into info table to signal if abp domains have been found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 451a40fc..39ac2ded 100755 --- a/gravity.sh +++ b/gravity.sh @@ -137,6 +137,18 @@ update_gravity_timestamp() { return 0 } +# Update timestamp when the gravity table was last updated successfully +set_abp_info() { + pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('abp_domains',${abp_domains});" + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to update ABP domain status in database ${gravityDBfile}\\n ${output}" + return 1 + fi + return 0 +} + # Import domains from file and store them in the specified database table database_table_from_file() { # Define locals @@ -519,6 +531,10 @@ gravity_DownloadBlocklists() { gravity_Blackbody=true } + +# global variable to indicate if we found ABP style domains during the gravity run +# is saved in gravtiy's info table to signal FTL if such domains are available +abp_domains=0 parseList() { local adlistID="${1}" src="${2}" target="${3}" temp_file temp_file_base non_domains sample_non_domains @@ -536,9 +552,10 @@ parseList() { # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already sed -r "/^([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/!d" "${src}" > "${temp_file}" - # if there is at least one ABP style domains + # if there is at least one ABP style domain if grep -E "^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" -m 1 -q "${src}"; then echo " ${INFO} List contained AdBlock Plus style domains" + abp_domains=1 # 2. Add all supported ABP style lines (||subdomain.domain.tlp^) sed -r "/^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\^$/!d" "${src}" >> "${temp_file}" fi @@ -1014,6 +1031,9 @@ fi # Update gravity timestamp update_gravity_timestamp +# Set abp_domain info field +set_abp_info + # Ensure proper permissions are set for the database chown pihole:pihole "${gravityDBfile}" chmod g+w "${piholeDir}" "${gravityDBfile}" From 16385af3ef473e4f148224bfc735e2582e666a3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 26 Feb 2023 21:16:45 +0100 Subject: [PATCH 008/462] Use dedicated pattern variable to make RegEx reusable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 39ac2ded..8db728a4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -536,7 +536,7 @@ gravity_DownloadBlocklists() { # is saved in gravtiy's info table to signal FTL if such domains are available abp_domains=0 parseList() { - local adlistID="${1}" src="${2}" target="${3}" temp_file temp_file_base non_domains sample_non_domains + local adlistID="${1}" src="${2}" target="${3}" temp_file temp_file_base non_domains sample_non_domains valid_domain_pattern abp_domain_pattern # Create a temporary file for the sed magic instead of using "${target}" directly # this allows to split the sed commands to improve readability @@ -547,17 +547,23 @@ parseList() { rm "${temp_file_base}" temp_file="/proc/$$/fd/3" - - # 1. Add all valid domains (adapted from https://stackoverflow.com/a/30007882) + # define valid domain patterns # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already - sed -r "/^([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/!d" "${src}" > "${temp_file}" + # adapted from https://stackoverflow.com/a/30007882 + # supported ABP style: ||subdomain.domain.tlp^ - # if there is at least one ABP style domain - if grep -E "^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" -m 1 -q "${src}"; then + valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" + abp_domain_pattern="\|\|${valid_domain_pattern}\^" + + + # 1. Add all valid domains + sed -r "/^${valid_domain_pattern}$/!d" "${src}" > "${temp_file}" + + # 2. Add valid ABP style domains if there is at least one such domain + if grep -E "^${abp_domain_pattern}$" -m 1 -q "${src}"; then echo " ${INFO} List contained AdBlock Plus style domains" abp_domains=1 - # 2. Add all supported ABP style lines (||subdomain.domain.tlp^) - sed -r "/^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\^$/!d" "${src}" >> "${temp_file}" + sed -r "/^${abp_domain_pattern}$/!d" "${src}" >> "${temp_file}" fi From 73de49323c13f1e87f68e272e9a71e12d03e3f65 Mon Sep 17 00:00:00 2001 From: MichaIng Date: Tue, 21 Feb 2023 21:49:03 +0100 Subject: [PATCH 009/462] Remove systemd service and optionally override configs on uninstall This has been forgotten when adding the new native systemd service. Signed-off-by: MichaIng --- automated install/uninstall.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index c36027fc..7a1a290d 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -193,6 +193,18 @@ removeNoPurge() { else service pihole-FTL stop fi + ${SUDO} rm -f /etc/systemd/system/pihole-FTL.service + if [[ -d '/etc/systemd/system/pihole-FTL.service.d' ]]; then + read -rp " ${QST} FTL service override directory /etc/systemd/system/pihole-FTL.service.d detected. Do you wish to remove this from your system? [y/N] " answer + case $answer in + [yY]*) + echo -ne " ${INFO} Removing /etc/systemd/system/pihole-FTL.service.d..." + ${SUDO} rm -R /etc/systemd/system/pihole-FTL.service.d + echo -e "${OVER} ${INFO} Removed /etc/systemd/system/pihole-FTL.service.d" + ;; + *) echo -e " ${INFO} Leaving /etc/systemd/system/pihole-FTL.service.d in place.";; + esac + fi ${SUDO} rm -f /etc/init.d/pihole-FTL ${SUDO} rm -f /usr/bin/pihole-FTL echo -e "${OVER} ${TICK} Removed pihole-FTL" From ddf972cede32c3b80fc25cfdab356c101d08f36e Mon Sep 17 00:00:00 2001 From: Alex Date: Thu, 8 Dec 2022 20:00:33 +0200 Subject: [PATCH 010/462] build: harden workflow permissions Signed-off-by: Alex --- .github/workflows/sync-back-to-dev.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index f689ae36..8572ffde 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -5,8 +5,30 @@ on: branches: - master +# The section is needed to drop the default write-all permissions for all jobs +# that are granted on `push` event. By specifying any permission explicitly +# all others are set to none. By using the principle of least privilege the damage a compromised +# workflow can do (because of an injection or compromised third party tool or +# action) is restricted. Adding labels to issues, commenting +# on pull-requests, etc. may need additional permissions: +# +# Syntax for this section: +# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions +# +# Reference for how to assign permissions on a job-by-job basis: +# https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs +# +# Reference for available permissions that we can enable if needed: +# https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token +permissions: {} + jobs: sync-branches: + # The job needs to be able to pull the code and create a pull request. + permissions: + contents: read # for actions/checkout + pull-requests: write # to create pull request + runs-on: ubuntu-latest name: Syncing branches steps: From 0b5da9f0dabccaf902aada60527c3a4b5855b925 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 4 Mar 2023 10:36:07 +0100 Subject: [PATCH 011/462] Allow final dot (root zone) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 8db728a4..281d5d54 100755 --- a/gravity.sh +++ b/gravity.sh @@ -552,7 +552,7 @@ parseList() { # adapted from https://stackoverflow.com/a/30007882 # supported ABP style: ||subdomain.domain.tlp^ - valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" + valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\.?" abp_domain_pattern="\|\|${valid_domain_pattern}\^" From 0b60601f863fff2453b5a1e9cb9f678d33e72b13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 4 Mar 2023 10:57:42 +0000 Subject: [PATCH 012/462] Bump pytest from 7.2.1 to 7.2.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.2.1 to 7.2.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.2.1...7.2.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index c7848e8d..e64e8c66 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ docker-compose == 1.29.2 -pytest == 7.2.1 +pytest == 7.2.2 pytest-xdist == 3.2.0 pytest-testinfra == 7.0.0 tox == 4.4.6 From d6f5552ccf4ee96c3ca137184b99307cf81b1496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 6 Mar 2023 21:16:51 +0100 Subject: [PATCH 013/462] Convert domain to lowercase in pihole -q MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/query.sh | 57 ++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index d48e9363..4061e17c 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -30,33 +30,6 @@ gravityDBfile="${GRAVITYDB}" colfile="/opt/pihole/COL_TABLE" source "${colfile}" -# Scan an array of files for matching strings -scanList(){ - # Escape full stops - local domain="${1}" esc_domain="${1//./\\.}" lists="${2}" list_type="${3:-}" - - # Prevent grep from printing file path - cd "$piholeDir" || exit 1 - - # Prevent grep -i matching slowly: https://bit.ly/2xFXtUX - export LC_CTYPE=C - - # /dev/null forces filename to be printed when only one list has been generated - case "${list_type}" in - "exact" ) grep -i -E -l "(^|(?/dev/null;; - # Iterate through each regexp and check whether it matches the domainQuery - # If it does, print the matching regexp and continue looping - # Input 1 - regexps | Input 2 - domainQuery - "regex" ) - for list in ${lists}; do - if [[ "${domain}" =~ ${list} ]]; then - printf "%b\n" "${list}"; - fi - done;; - * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; - esac -} - if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then echo "Usage: pihole -q [option] Example: 'pihole -q -exact domain.com' @@ -88,11 +61,41 @@ case "${options}" in * ) domainQuery="${options}";; esac +# convert the domain to lowercase +domainQuery=$(echo "${domainQuery}" | tr '[:upper:]' '[:lower:]') + if [[ -n "${str:-}" ]]; then echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information." exit 1 fi +# Scan an array of files for matching strings +scanList(){ + # Escape full stops + local domain="${1}" esc_domain="${1//./\\.}" lists="${2}" list_type="${3:-}" + + # Prevent grep from printing file path + cd "$piholeDir" || exit 1 + + # Prevent grep -i matching slowly: https://bit.ly/2xFXtUX + export LC_CTYPE=C + + # /dev/null forces filename to be printed when only one list has been generated + case "${list_type}" in + "exact" ) grep -i -E -l "(^|(?/dev/null;; + # Iterate through each regexp and check whether it matches the domainQuery + # If it does, print the matching regexp and continue looping + # Input 1 - regexps | Input 2 - domainQuery + "regex" ) + for list in ${lists}; do + if [[ "${domain}" =~ ${list} ]]; then + printf "%b\n" "${list}"; + fi + done;; + * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; + esac +} + scanDatabaseTable() { local domain table list_type querystr result extra domain="$(printf "%q" "${1}")" From 309ee789036708b77453e2c71f1a2a22180d85ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 7 Mar 2023 20:40:16 +0100 Subject: [PATCH 014/462] Use distinct variabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/query.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 4061e17c..99c1cf0f 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -57,12 +57,12 @@ options=$(sed -E 's/ ?-(all|exact) ?//g' <<< "${options}") case "${options}" in "" ) str="No domain specified";; *" "* ) str="Unknown query option specified";; - *[![:ascii:]]* ) domainQuery=$(idn2 "${options}");; - * ) domainQuery="${options}";; + *[![:ascii:]]* ) rawDomainQuery=$(idn2 "${options}");; + * ) rawDomainQuery="${options}";; esac # convert the domain to lowercase -domainQuery=$(echo "${domainQuery}" | tr '[:upper:]' '[:lower:]') +domainQuery=$(echo "${rawDomainQuery}" | tr '[:upper:]' '[:lower:]') if [[ -n "${str:-}" ]]; then echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information." @@ -82,7 +82,7 @@ scanList(){ # /dev/null forces filename to be printed when only one list has been generated case "${list_type}" in - "exact" ) grep -i -E -l "(^|(?/dev/null;; + "exact" ) grep -i -E -l "(^|(?/dev/null;; # Iterate through each regexp and check whether it matches the domainQuery # If it does, print the matching regexp and continue looping # Input 1 - regexps | Input 2 - domainQuery @@ -92,7 +92,7 @@ scanList(){ printf "%b\n" "${list}"; fi done;; - * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; + * ) grep -i "${esc_domain}" "${lists}" /dev/null 2>/dev/null;; esac } From 71e262c37f5582d54e5c7d9f05b4f61533ca81a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 14 Mar 2023 19:42:05 +0100 Subject: [PATCH 015/462] Revert "Allow final dot (root zone)" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 0b5da9f0dabccaf902aada60527c3a4b5855b925. Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 2 +- gravity.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index ad25e866..fa1cebbb 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -230,7 +230,7 @@ initialize_debug() { # This is a function for visually displaying the current test that is being run. # Accepts one variable: the name of what is being diagnosed -# Colors do not show in the dasboard, but the icons do: [i], [✓], and [✗] +# Colors do not show in the dashboard, but the icons do: [i], [✓], and [✗] echo_current_diagnostic() { # Colors are used for visually distinguishing each test in the output # These colors do not show in the GUI, but the formatting will diff --git a/gravity.sh b/gravity.sh index 281d5d54..8db728a4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -552,7 +552,7 @@ parseList() { # adapted from https://stackoverflow.com/a/30007882 # supported ABP style: ||subdomain.domain.tlp^ - valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\.?" + valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" abp_domain_pattern="\|\|${valid_domain_pattern}\^" From c35ed6805159e3df319d0bb663cc145d5987a8af Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Mon, 27 Feb 2023 01:46:12 -0300 Subject: [PATCH 016/462] Allow `pihole -q` matching ABP subdomains Signed-off-by: RD WebDesign --- advanced/Scripts/query.sh | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 463b0901..71309b56 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -102,6 +102,16 @@ scanDatabaseTable() { table="${2}" list_type="${3:-}" + # Create search string for ABP entries + local abpentry="${domain}" searchstr + + searchstr="'||${abpentry}^'" + while [ "${abpentry}" != "${abpentry/./}" ] + do + abpentry=$(echo "${abpentry}" | cut -f 2- -d '.') + searchstr=$(echo "$searchstr, '||${abpentry}^'") + done + # As underscores are legitimate parts of domains, we escape them when using the LIKE operator. # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched @@ -109,12 +119,12 @@ scanDatabaseTable() { if [[ "${table}" == "gravity" ]]; then case "${exact}" in "exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";; - * ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + * ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE (domain IN (${searchstr}) OR domain LIKE '%${domain//_/\\_}%' ESCAPE '\\')";; esac else case "${exact}" in "exact" ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain = '${domain}'";; - * ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + * ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND (domain IN (${searchstr}) OR domain LIKE '%${domain//_/\\_}%' ESCAPE '\\')";; esac fi From 20f8c6af3c3b10fa4be6bd23365ecd7b89c93cea Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Tue, 28 Feb 2023 15:55:02 -0300 Subject: [PATCH 017/462] Search for ABP entries only if they exist in gravity.db and use `abp_domains` property. Signed-off-by: RD WebDesign --- advanced/Scripts/query.sh | 40 +++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 71309b56..c6a932b7 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -102,29 +102,45 @@ scanDatabaseTable() { table="${2}" list_type="${3:-}" - # Create search string for ABP entries - local abpentry="${domain}" searchstr - - searchstr="'||${abpentry}^'" - while [ "${abpentry}" != "${abpentry/./}" ] - do - abpentry=$(echo "${abpentry}" | cut -f 2- -d '.') - searchstr=$(echo "$searchstr, '||${abpentry}^'") - done - # As underscores are legitimate parts of domains, we escape them when using the LIKE operator. # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. if [[ "${table}" == "gravity" ]]; then + local abpquerystr, abpfound, abpentry, searchstr + + # Are there ABP entries on gravity? + # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set + abpquerystr="SELECT EXISTS (SELECT 1 FROM info WHERE property='abp_domains' and value='1')" + abpfound="$(pihole-FTL sqlite3 "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null + + # Create search string for ABP entries only if needed + if [ "${abpfound}" -eq 1 ]; then + abpentry="${domain}" + + searchstr="'||${abpentry}^'" + + # While a dot is found ... + while [ "${abpentry}" != "${abpentry/./}" ] + do + # ... remove text before the dot (including the dot) and append the result to $searchstr + abpentry=$(echo "${abpentry}" | cut -f 2- -d '.') + searchstr="$searchstr, '||${abpentry}^'" + done + + # The final search string will look like: + # "domain IN ('||sub2.sub1.domain.com^', '||sub1.domain.com^', '||domain.com^', '||com^') OR" + searchstr="domain IN (${searchstr}) OR " + fi + case "${exact}" in "exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";; - * ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE (domain IN (${searchstr}) OR domain LIKE '%${domain//_/\\_}%' ESCAPE '\\')";; + * ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE ${searchstr} domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; esac else case "${exact}" in "exact" ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain = '${domain}'";; - * ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND (domain IN (${searchstr}) OR domain LIKE '%${domain//_/\\_}%' ESCAPE '\\')";; + * ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; esac fi From b9a6970bfd04b8fbe9f78bc849b5bddc776202eb Mon Sep 17 00:00:00 2001 From: William Blew Date: Thu, 16 Mar 2023 19:36:22 -0700 Subject: [PATCH 018/462] Fix addKey to handle substrings of existing keys Fix addKey to handle the case where a key is being added, and that key is the leading substring of an already existing key within that file. For example: add "server=192.168.1.1", when "server=192.168.1.178" already exists within the /etc/dnsmasq.d/01-pihole.conf file. Check pihole docker with PIHOLE_DNS="192.168.1.178;192.168.1.1". Its /etc/dnsmasq/01-pihole.conf will be missing its second server= entry. Add the test_key_addition_substr, to test addKey when its adding a substring key of an existing key in the file. Signed-off-by: William Blew --- advanced/Scripts/utils.sh | 2 +- test/test_any_utils.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 37516472..9e714606 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -57,7 +57,7 @@ addKey(){ # touch file to prevent grep error if file does not exist yet touch "${file}" - if ! grep -q "^${key}" "${file}"; then + if ! grep -q "^${key}$" "${file}"; then # Key does not exist, add it. echo "${key}" >> "${file}" fi diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 5b4075d9..b3fabe6c 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -40,6 +40,26 @@ def test_key_addition_works(host): assert expected_stdout == output.stdout +def test_key_addition_substr(host): + """Confirms addKey adds substring keys (no value) to a file""" + host.run( + """ + source /opt/pihole/utils.sh + addKey "./testoutput" "KEY_ONE" + addKey "./testoutput" "KEY_O" + addKey "./testoutput" "KEY_TWO" + addKey "./testoutput" "Y_TWO" + """ + ) + output = host.run( + """ + cat ./testoutput + """ + ) + expected_stdout = "KEY_ONE\nKEY_O\nKEY_TWO\nY_TWO\n" + assert expected_stdout == output.stdout + + def test_key_removal_works(host): """Confirms removeKey removes a key or key/value pair""" host.run( From 3c91b6558dcc947736e1df1631a9a7dfc7d32f9b Mon Sep 17 00:00:00 2001 From: William Blew Date: Fri, 17 Mar 2023 11:47:26 -0700 Subject: [PATCH 019/462] restore the addKey comment, reworded for anchors Per @dschaper, restore the addKey clarifying comment. It has been reworded to describe the use of anchors where before it referenced using grep's 'match only an entire line' argument. Signed-off-by: William Blew --- advanced/Scripts/utils.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 9e714606..f655e56c 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -57,6 +57,10 @@ addKey(){ # touch file to prevent grep error if file does not exist yet touch "${file}" + # Match key against entire line, using both anchors. We assume + # that the file's keys never have bounding whitespace. Anchors + # are necessary to ensure the key is considered absent when it + # is a substring of another key present in the file. if ! grep -q "^${key}$" "${file}"; then # Key does not exist, add it. echo "${key}" >> "${file}" From 3a592e56ba6b6e41d11f8fd3769a42892e4a3059 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Mar 2023 10:56:52 +0000 Subject: [PATCH 020/462] Bump actions/checkout from 3.3.0 to 3.4.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.3.0 to 3.4.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.3.0...v3.4.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5539cec9..01be8b25 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 25c3a7f7..0fe850d1 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ce948e09..27867ef3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 - name: Check scripts in repository are executable run: | @@ -62,7 +62,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.3.0 + uses: actions/checkout@v3.4.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 From 8a2829de874aa1c90fba1ee38bfe44f5b480d3fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Mar 2023 10:58:49 +0000 Subject: [PATCH 021/462] Bump pytest-xdist from 3.2.0 to 3.2.1 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.2.0 to 3.2.1. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.2.0...v3.2.1) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index e64e8c66..b2f0e8c4 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ docker-compose == 1.29.2 pytest == 7.2.2 -pytest-xdist == 3.2.0 +pytest-xdist == 3.2.1 pytest-testinfra == 7.0.0 tox == 4.4.6 From 686da5a9480fc1791c0623f7a035c84dceb23a06 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Mar 2023 10:58:57 +0000 Subject: [PATCH 022/462] Bump tox from 4.4.6 to 4.4.7 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.4.6 to 4.4.7. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.4.6...4.4.7) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index e64e8c66..ff2f49d0 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.2.2 pytest-xdist == 3.2.0 pytest-testinfra == 7.0.0 -tox == 4.4.6 +tox == 4.4.7 From 6b919f3a2e36f724e8dd2a8f3b4e518d28de20a1 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Tue, 21 Mar 2023 12:27:20 -0300 Subject: [PATCH 023/462] Removing unnecessary commas Signed-off-by: RD WebDesign --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index c6a932b7..604ac2ed 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -107,7 +107,7 @@ scanDatabaseTable() { # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. if [[ "${table}" == "gravity" ]]; then - local abpquerystr, abpfound, abpentry, searchstr + local abpquerystr abpfound abpentry searchstr # Are there ABP entries on gravity? # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set From 66ed7c9ea374cc7c322807b6a0fc5cb33ac45d80 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Tue, 21 Mar 2023 16:34:50 -0300 Subject: [PATCH 024/462] Declaring all local variables under the function declaration line Signed-off-by: RD WebDesign --- advanced/Scripts/query.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 604ac2ed..12295fbc 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -97,7 +97,7 @@ scanList(){ } scanDatabaseTable() { - local domain table list_type querystr result extra + local domain table list_type querystr result extra abpquerystr abpfound abpentry searchstr domain="$(printf "%q" "${1}")" table="${2}" list_type="${3:-}" @@ -107,7 +107,6 @@ scanDatabaseTable() { # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. if [[ "${table}" == "gravity" ]]; then - local abpquerystr abpfound abpentry searchstr # Are there ABP entries on gravity? # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set From 76b7453f902c50364a0ca688ef06eb106116f0f4 Mon Sep 17 00:00:00 2001 From: kot0dama <89980752+kot0dama@users.noreply.github.com> Date: Sun, 19 Mar 2023 05:32:46 +0100 Subject: [PATCH 025/462] Add configurable GRAVITY_TMPDIR variable into setupVars MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Loïc Gomez <89980752+kot0dama@users.noreply.github.com> --- gravity.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 8db728a4..ca859e38 100755 --- a/gravity.sh +++ b/gravity.sh @@ -52,6 +52,14 @@ else exit 1 fi +# Set up tmp dir variable in case it's not configured +: "${GRAVITY_TMPDIR:=/tmp}" + +if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then + echo -e " ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}" + GRAVITY_TMPDIR="/tmp" +fi + # Source pihole-FTL from install script pihole_FTL="${piholeDir}/pihole-FTL.conf" if [[ -f "${pihole_FTL}" ]]; then @@ -157,7 +165,7 @@ database_table_from_file() { src="${2}" backup_path="${piholeDir}/migration_backup" backup_file="${backup_path}/$(basename "${2}")" - tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")" + tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".gravity")" local timestamp timestamp="$(date --utc +'%s')" @@ -430,7 +438,7 @@ gravity_DownloadBlocklists() { echo -e "${OVER} ${TICK} ${str}" fi - target="$(mktemp -p "/tmp" --suffix=".gravity")" + target="$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".gravity")" # Use compression to reduce the amount of data that is transferred # between the Pi-hole and the ad list provider. Use this feature @@ -643,7 +651,7 @@ gravity_DownloadBlocklistFromUrl() { local heisenbergCompensator="" patternBuffer str httpCode success="" ip # Create temp file to store content on disk instead of RAM - patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb") + patternBuffer=$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".phgpb") # Determine if $saveLocation has read permission if [[ -r "${saveLocation}" && $url != "file"* ]]; then @@ -860,7 +868,7 @@ gravity_Cleanup() { # Delete tmp content generated by Gravity rm ${piholeDir}/pihole.*.txt 2> /dev/null rm ${piholeDir}/*.tmp 2> /dev/null - rm /tmp/*.phgpb 2> /dev/null + rm "${GRAVITY_TMPDIR}"/*.phgpb 2> /dev/null # Ensure this function only runs when gravity_SetDownloadOptions() has completed if [[ "${gravity_Blackbody:-}" == true ]]; then From 58275ecd132d6f56529592baf02b207ee89f73dc Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 22 Mar 2023 21:52:39 +0000 Subject: [PATCH 026/462] Revert "Ignore commented lines when reading PRIVACYLEVEL from config file" --- advanced/Scripts/utils.sh | 22 ++-------------------- automated install/basic-install.sh | 3 +-- test/test_any_utils.py | 16 ---------------- 3 files changed, 3 insertions(+), 38 deletions(-) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 2d3c7fb1..f655e56c 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -44,7 +44,7 @@ addOrEditKeyValPair() { } ####################### -# Takes two arguments: file and key. +# Takes two arguments: file, and key. # Adds a key to target file # # Example usage: @@ -68,7 +68,7 @@ addKey(){ } ####################### -# Takes two arguments: file and key. +# Takes two arguments: file, and key. # Deletes a key or key/value pair from target file # # Example usage: @@ -80,24 +80,6 @@ removeKey() { sed -i "/^${key}/d" "${file}" } -####################### -# Takes two arguments: file and key. -# Returns the value of a given key from target file -# - ignores all commented lines -# - only returns the first value if multiple identical keys exist -# -# -# Example usage: -# getVal "/etc/pihole/setupVars.conf" "PIHOLE_DNS_1" -####################### -getVal() { - local file="${1}" - local key="${2}" - local value - value=$(sed -e '/^[[:blank:]]*#/d' "${file}" | grep "${key}" | awk -F "=" 'NR==1{printf$2}') - printf "%s" "$value" -} - ####################### # returns FTL's current telnet API port based on the setting in /etc/pihole-FTL.conf diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a781f8c6..ccb5eac7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2612,8 +2612,7 @@ main() { # Get the privacy level if it exists (default is 0) if [[ -f "${FTL_CONFIG_FILE}" ]]; then - # use getVal from utils.sh to get PRIVACYLEVEL - PRIVACY_LEVEL=$(getVal "${FTL_CONFIG_FILE}" "PRIVACYLEVEL") + PRIVACY_LEVEL=$(sed -ne 's/PRIVACYLEVEL=\(.*\)/\1/p' "${FTL_CONFIG_FILE}") # If no setting was found, default to 0 PRIVACY_LEVEL="${PRIVACY_LEVEL:-0}" diff --git a/test/test_any_utils.py b/test/test_any_utils.py index cb5ddeaa..b3fabe6c 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -82,22 +82,6 @@ def test_key_removal_works(host): assert expected_stdout == output.stdout -def test_get_value_works(host): - """Confirms getVal returns the correct value for a given key""" - output = host.run( - """ - source /opt/pihole/utils.sh - echo "Somekey=xxx" >> /tmp/testfile - echo "#Testkey=1234" >> /tmp/testfile - echo "Testkey=5678" >> /tmp/testfile - echo "Testkey=abcd" >> /tmp/testfile - getVal "/tmp/testfile" "Testkey" - """ - ) - expected_stdout = "5678" - assert expected_stdout == output.stdout - - def test_getFTLAPIPort_default(host): """Confirms getFTLAPIPort returns the default API port""" output = host.run( From c96463bda20ca261286c7b4e7dc2a5dbc1103723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 22 Mar 2023 22:56:00 +0100 Subject: [PATCH 027/462] Fix getting 'privacylevel' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ccb5eac7..24fe9e54 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2612,7 +2612,8 @@ main() { # Get the privacy level if it exists (default is 0) if [[ -f "${FTL_CONFIG_FILE}" ]]; then - PRIVACY_LEVEL=$(sed -ne 's/PRIVACYLEVEL=\(.*\)/\1/p' "${FTL_CONFIG_FILE}") + # get the value from $FTL_CONFIG_FILE (and ignoring all commented lines) + PRIVACY_LEVEL=$(sed -e '/^[[:blank:]]*#/d' "${FTL_CONFIG_FILE}" | grep "PRIVACYLEVEL" | awk -F "=" 'NR==1{printf$2}') # If no setting was found, default to 0 PRIVACY_LEVEL="${PRIVACY_LEVEL:-0}" From fa116389c25ad45d284e8633aa7f9395599639af Mon Sep 17 00:00:00 2001 From: ipitio <21136719+ipitio@users.noreply.github.com> Date: Thu, 23 Mar 2023 01:23:35 +0000 Subject: [PATCH 028/462] remove old comments Signed-off-by: ipitio <21136719+ipitio@users.noreply.github.com> --- advanced/Scripts/piholeDebug.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index fa1cebbb..6f747855 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -230,10 +230,8 @@ initialize_debug() { # This is a function for visually displaying the current test that is being run. # Accepts one variable: the name of what is being diagnosed -# Colors do not show in the dashboard, but the icons do: [i], [✓], and [✗] echo_current_diagnostic() { # Colors are used for visually distinguishing each test in the output - # These colors do not show in the GUI, but the formatting will log_write "\\n${COL_PURPLE}*** [ DIAGNOSING ]:${COL_NC} ${1}" } From 95b12bad34a3fe719866f4f96cdbe9cea12ee039 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Mar 2023 10:56:35 +0000 Subject: [PATCH 029/462] Bump actions/stale from 7.0.0 to 8.0.0 Bumps [actions/stale](https://github.com/actions/stale) from 7.0.0 to 8.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v7.0.0...v8.0.0) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- .github/workflows/stale_pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 58a2e647..071746a8 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -13,7 +13,7 @@ jobs: issues: write steps: - - uses: actions/stale@v7.0.0 + - uses: actions/stale@v8.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 diff --git a/.github/workflows/stale_pr.yml b/.github/workflows/stale_pr.yml index c45e3cb7..2db2a25d 100644 --- a/.github/workflows/stale_pr.yml +++ b/.github/workflows/stale_pr.yml @@ -17,7 +17,7 @@ jobs: pull-requests: write steps: - - uses: actions/stale@v7.0.0 + - uses: actions/stale@v8.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Do not automatically mark PR/issue as stale From 0656ceb149a25bb424f654da74ae0f34e6b4efd0 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 24 Mar 2023 23:15:49 +0000 Subject: [PATCH 030/462] Speed things up a bit with some humble greps. Consolodate regexes and remove the need for so many mapfile/arrays Signed-off-by: Adam Warner Use temp files for parsing and remove when done. Always rm the non-domains temp file. exit 1 if gravity database creation fails. Signed-off-by: Dan Schaper Co-authored-by: Dan Schaper Co-authored-by: DL6ER Co-authored-by: Adam Warner --- gravity.sh | 94 ++++++++++++++++++++---------------------------------- 1 file changed, 35 insertions(+), 59 deletions(-) diff --git a/gravity.sh b/gravity.sh index ca859e38..a415aa01 100755 --- a/gravity.sh +++ b/gravity.sh @@ -546,15 +546,6 @@ abp_domains=0 parseList() { local adlistID="${1}" src="${2}" target="${3}" temp_file temp_file_base non_domains sample_non_domains valid_domain_pattern abp_domain_pattern - # Create a temporary file for the sed magic instead of using "${target}" directly - # this allows to split the sed commands to improve readability - # we use a file handle here and remove the temporary file immediately so the content will be deleted in any case - # when the script stops - temp_file_base="$(mktemp -p "/tmp" --suffix=".gravity")" - exec 3>"$temp_file_base" - rm "${temp_file_base}" - temp_file="/proc/$$/fd/3" - # define valid domain patterns # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already # adapted from https://stackoverflow.com/a/30007882 @@ -563,62 +554,39 @@ parseList() { valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" abp_domain_pattern="\|\|${valid_domain_pattern}\^" - - # 1. Add all valid domains - sed -r "/^${valid_domain_pattern}$/!d" "${src}" > "${temp_file}" - - # 2. Add valid ABP style domains if there is at least one such domain - if grep -E "^${abp_domain_pattern}$" -m 1 -q "${src}"; then - echo " ${INFO} List contained AdBlock Plus style domains" - abp_domains=1 - sed -r "/^${abp_domain_pattern}$/!d" "${src}" >> "${temp_file}" - fi - - - # Find lines containing no domains or with invalid characters (not matching regex above) - # This is simply everything that is not in $temp_file compared to $src - # Remove duplicates from the list - mapfile -t non_domains < <(grep -Fvf "${temp_file}" "${src}" | sort -u ) - - # 3. Remove trailing period (see https://github.com/pi-hole/pi-hole/issues/4701) - # 4. Append ,adlistID to every line - # 5. Ensures there is a newline on the last line - # and write everything to the target file - sed "s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${temp_file}" >> "${target}" - # A list of items of common local hostnames not to report as unusable # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files # but flagging them as unusable causes more confusion than it's worth - so we suppress them from the output false_positives="localhost|localhost.localdomain|local|broadcasthost|localhost|ip6-localhost|ip6-loopback|lo0 localhost|ip6-localnet|ip6-mcastprefix|ip6-allnodes|ip6-allrouters|ip6-allhosts" - # if there are any non-domains, filter the array for false-positives - # Credit: https://stackoverflow.com/a/40264051 - if [[ "${#non_domains[@]}" -gt 0 ]]; then - mapfile -d $'\0' -t non_domains < <(printf '%s\0' "${non_domains[@]}" | grep -Ezv "^${false_positives}") + # Extract valid domains from source file and append ,${adlistID} to each line and save count to variable for display. + num_domains=$(grep -E "^(${valid_domain_pattern}|${abp_domain_pattern})$" "${src}" | tee >(sed "s/$/,${adlistID}/" >> "${target}") | wc -l) + + # Check if the source file contained AdBlock Plus style domains, if so we set the global variable and inform the user + if grep -E "^${abp_domain_pattern}$" -m 1 -q "${src}"; then + echo " ${INFO} List contained AdBlock Plus style domains" + abp_domains=1 fi - # Get a sample of non-domain entries, limited to 5 (the list should already have been de-duplicated) - IFS=" " read -r -a sample_non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | head -n 5 | tr '\n' ' ')" + # For completeness, we will get a count of non_domains (this is the number of entries left after stripping the source of comments/duplicates/false positives/domains) + invalid_domains="$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".ph-non-domains")" - # Get the number of domains added - num_domains="$(grep -c "^" "${temp_file}")" - # Get the number of non_domains (this is the number of entries left after stripping the source of comments/duplicates/false positives/domains) - num_non_domains="${#non_domains[@]}" + num_non_domains=$(grep -Ev "^(${valid_domain_pattern}|${abp_domain_pattern}|${false_positives})$" "${src}" | tee "${invalid_domains}" | wc -l) # If there are unusable lines, we display some information about them. This is not error or major cause for concern. if [[ "${num_non_domains}" -ne 0 ]]; then - echo " ${INFO} Imported ${num_domains} domains, ignoring ${num_non_domains} non-domain entries" + type="domains" + if [[ "${abp_domains}" -ne 0 ]]; then + type="patterns" + fi + echo " ${INFO} Imported ${num_domains} ${type}, ignoring ${num_non_domains} non-domain entries" echo " Sample of non-domain entries:" - for each in "${sample_non_domains[@]}" - do - echo " - ${each}" - done + invalid_lines=$(head -n 5 "${invalid_domains}") + echo "${invalid_lines}" | awk '{print " - " $0}' else echo " ${INFO} Imported ${num_domains} domains" fi - - # close file handle - exec 3<&- + rm "${invalid_domains}" } compareLists() { @@ -648,10 +616,10 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { local url="${1}" cmd_ext="${2}" agent="${3}" adlistID="${4}" saveLocation="${5}" target="${6}" compression="${7}" - local heisenbergCompensator="" patternBuffer str httpCode success="" ip + local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip # Create temp file to store content on disk instead of RAM - patternBuffer=$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".phgpb") + listCurlBuffer=$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".phgpb") # Determine if $saveLocation has read permission if [[ -r "${saveLocation}" && $url != "file"* ]]; then @@ -705,12 +673,12 @@ gravity_DownloadBlocklistFromUrl() { fi # shellcheck disable=SC2086 - httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${patternBuffer}" 2> /dev/null) + httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) case $url in # Did we "download" a local file? "file"*) - if [[ -s "${patternBuffer}" ]]; then + if [[ -s "${listCurlBuffer}" ]]; then echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true else echo -e "${OVER} ${CROSS} ${str} Not found / empty list" @@ -743,10 +711,12 @@ gravity_DownloadBlocklistFromUrl() { database_adlist_status "${adlistID}" "2" database_adlist_number "${adlistID}" done="true" - # Check if $patternbuffer is a non-zero length file - elif [[ -s "${patternBuffer}" ]]; then + # Check if $listCurlBuffer is a non-zero length file + elif [[ -s "${listCurlBuffer}" ]]; then # Determine if blocklist is non-standard and parse as appropriate - gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" + gravity_ParseFileIntoDomains "${listCurlBuffer}" "${saveLocation}" + # Remove curl buffer file after its use + rm "${listCurlBuffer}" # Add domains to database table file parseList "${adlistID}" "${saveLocation}" "${target}" # Compare lists, are they identical? @@ -756,7 +726,7 @@ gravity_DownloadBlocklistFromUrl() { database_adlist_number "${adlistID}" done="true" else - # Fall back to previously cached list if $patternBuffer is empty + # Fall back to previously cached list if $listCurlBuffer is empty echo -e " ${INFO} Received empty file" fi fi @@ -868,7 +838,10 @@ gravity_Cleanup() { # Delete tmp content generated by Gravity rm ${piholeDir}/pihole.*.txt 2> /dev/null rm ${piholeDir}/*.tmp 2> /dev/null + # listCurlBuffer location rm "${GRAVITY_TMPDIR}"/*.phgpb 2> /dev/null + # invalid_domains location + rm "${GRAVITY_TMPDIR}"/*.ph-non-domains 2> /dev/null # Ensure this function only runs when gravity_SetDownloadOptions() has completed if [[ "${gravity_Blackbody:-}" == true ]]; then @@ -1031,7 +1004,10 @@ if ! gravity_CheckDNSResolutionAvailable; then exit 1 fi -gravity_DownloadBlocklists +if ! gravity_DownloadBlocklists; then + echo -e " ${CROSS} Unable to create gravity database. Please try again later. If the problem persists, please contact support." + exit 1 +fi # Create local.list gravity_generateLocalList From c71460e4b69fc97b5ec5e50f5401ae4a9d12314c Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Tue, 28 Mar 2023 16:41:11 -0300 Subject: [PATCH 031/462] Allow TLD blocking using ABP style This validates patterns without dots (only for abp style), allowing TLDs to be blocked Signed-off-by: RD WebDesign --- gravity.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index a415aa01..19471cce 100755 --- a/gravity.sh +++ b/gravity.sh @@ -549,10 +549,12 @@ parseList() { # define valid domain patterns # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already # adapted from https://stackoverflow.com/a/30007882 - # supported ABP style: ||subdomain.domain.tlp^ valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" - abp_domain_pattern="\|\|${valid_domain_pattern}\^" + + # supported ABP style: ||subdomain.domain.tld^ + # allow TLD blocking using ABP style: ||tld^ + abp_domain_pattern="\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)*[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\^" # A list of items of common local hostnames not to report as unusable # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files From 32fb2e69ff26e057bc6754197b5d343889dc63e7 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Wed, 29 Mar 2023 00:02:42 -0300 Subject: [PATCH 032/462] Spliting the regex into TLD_pattern and subdomain_pattern Signed-off-by: RD WebDesign --- gravity.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 19471cce..43159378 100755 --- a/gravity.sh +++ b/gravity.sh @@ -550,11 +550,14 @@ parseList() { # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already # adapted from https://stackoverflow.com/a/30007882 - valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" + TLD_pattern="[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" + subdomain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)" + + valid_domain_pattern="${subdomain_pattern}+${TLD_pattern}" # supported ABP style: ||subdomain.domain.tld^ - # allow TLD blocking using ABP style: ||tld^ - abp_domain_pattern="\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)*[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\^" + # Subdomain is optional for ABP style, allowing TLD blocking: ||tld^ + abp_domain_pattern="\|\|${subdomain_pattern}*${TLD_pattern}\^" # A list of items of common local hostnames not to report as unusable # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files From 66bfa606a7d2b3abb5ddffb673e23db6d5028985 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Wed, 29 Mar 2023 14:17:41 -0300 Subject: [PATCH 033/462] Using a better text for the comment Signed-off-by: RD WebDesign --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 43159378..34cad67d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -556,7 +556,7 @@ parseList() { valid_domain_pattern="${subdomain_pattern}+${TLD_pattern}" # supported ABP style: ||subdomain.domain.tld^ - # Subdomain is optional for ABP style, allowing TLD blocking: ||tld^ + # ${subdomain_pattern} is optional for ABP style, allowing TLD blocking: ||tld^ abp_domain_pattern="\|\|${subdomain_pattern}*${TLD_pattern}\^" # A list of items of common local hostnames not to report as unusable From 61ff5b2c761b3c8c511cd2628758dcab626094d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 7 Apr 2023 09:44:31 +0200 Subject: [PATCH 034/462] Unifiy sed commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: RD WebDesign Signed-off-by: Christian König --- gravity.sh | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/gravity.sh b/gravity.sh index 34cad67d..6ec53038 100755 --- a/gravity.sh +++ b/gravity.sh @@ -768,25 +768,21 @@ gravity_ParseFileIntoDomains() { tr '[:upper:]' '[:lower:]' < "${src}" > "${destination}" # 2) Remove carriage returns - sed -i 's/\r$//' "${destination}" + # 3) Remove comments (text starting with "#", include possible spaces before the hash sign) + # 4) Remove lines starting with ! (ABP Comments) + # 5) Remove lines starting with [ (ABP Header) + # 6) Remove lines containing "/" + # 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) + # 8) Remove empty lines - # 3a) Remove comments (text starting with "#", include possible spaces before the hash sign) - sed -i 's/\s*#.*//g' "${destination}" - - # 3b) Remove lines starting with ! (ABP Comments) - sed -i 's/\s*!.*//g' "${destination}" - - # 3c) Remove lines starting with [ (ABP Header) - sed -i 's/\s*\[.*//g' "${destination}" - - # 4) Remove lines containing "/" - sed -i -r '/(\/).*$/d' "${destination}" - - # 5) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) - sed -i -r 's/^.*\s+//g' "${destination}" - - # 6) Remove empty lines - sed -i '/^$/d' "${destination}" + sed -i -r \ + -e 's/\r$//' \ + -e 's/\s*#.*//g' \ + -e 's/\s*!.*//g' \ + -e 's/\s*\[.*//g' \ + -e '/(\/).*$/d' \ + -e 's/^.*\s+//g' \ + -e '/^$/d' "${destination}" chmod 644 "${destination}" } From cc17fe18a9863d6149957728b5f3f3877cc15ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 7 Apr 2023 10:14:59 +0200 Subject: [PATCH 035/462] Remove lines with ABP extended CSS selector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Adam Warner Signed-off-by: Christian König --- gravity.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/gravity.sh b/gravity.sh index 6ec53038..6ba27119 100755 --- a/gravity.sh +++ b/gravity.sh @@ -768,18 +768,20 @@ gravity_ParseFileIntoDomains() { tr '[:upper:]' '[:lower:]' < "${src}" > "${destination}" # 2) Remove carriage returns - # 3) Remove comments (text starting with "#", include possible spaces before the hash sign) - # 4) Remove lines starting with ! (ABP Comments) - # 5) Remove lines starting with [ (ABP Header) - # 6) Remove lines containing "/" - # 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) - # 8) Remove empty lines + # 3) Remove lines starting with ! (ABP Comments) + # 4) Remove lines starting with [ (ABP Header) + # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") + # 6) Remove comments (text starting with "#", include possible spaces before the hash sign) + # 7) Remove lines containing "/" + # 8) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) + # 9) Remove empty lines sed -i -r \ -e 's/\r$//' \ - -e 's/\s*#.*//g' \ -e 's/\s*!.*//g' \ -e 's/\s*\[.*//g' \ + -e '/\#[!|?|@]{0,1}\#/d' \ + -e 's/\s*#.*//g' \ -e '/(\/).*$/d' \ -e 's/^.*\s+//g' \ -e '/^$/d' "${destination}" From 2a0f72015364979fb1955e02bdb27ed13a62efe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 7 Apr 2023 10:25:25 +0200 Subject: [PATCH 036/462] Don't delete lines containing `/` as they should count as invalid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 6ba27119..f978cd0a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -772,9 +772,8 @@ gravity_ParseFileIntoDomains() { # 4) Remove lines starting with [ (ABP Header) # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") # 6) Remove comments (text starting with "#", include possible spaces before the hash sign) - # 7) Remove lines containing "/" - # 8) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) - # 9) Remove empty lines + # 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) + # 8) Remove empty lines sed -i -r \ -e 's/\r$//' \ @@ -782,7 +781,6 @@ gravity_ParseFileIntoDomains() { -e 's/\s*\[.*//g' \ -e '/\#[!|?|@]{0,1}\#/d' \ -e 's/\s*#.*//g' \ - -e '/(\/).*$/d' \ -e 's/^.*\s+//g' \ -e '/^$/d' "${destination}" From 9c4e74ffa7ec1425820c22f0eca463277bfbea7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 7 Apr 2023 12:23:11 +0200 Subject: [PATCH 037/462] Remove special handling of pgl.yoyo.org MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 34cad67d..54b521eb 100755 --- a/gravity.sh +++ b/gravity.sh @@ -463,12 +463,6 @@ gravity_DownloadBlocklists() { # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-) agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" - # Provide special commands for blocklists which may need them - case "${domain}" in - "pgl.yoyo.org") cmd_ext="-d mimetype=plaintext -d hostformat=hosts";; - *) cmd_ext="";; - esac - echo -e " ${INFO} Target: ${url}" local regex check_url # Check for characters NOT allowed in URLs From aaf828117d6435fdb9885aa8940dee05275f7eb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 7 Apr 2023 12:36:50 +0200 Subject: [PATCH 038/462] Remove unecessary $cmd_ext MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 54b521eb..5cb68850 100755 --- a/gravity.sh +++ b/gravity.sh @@ -421,7 +421,7 @@ gravity_DownloadBlocklists() { unset sources fi - local url domain agent cmd_ext str target compression + local url domain agent str target compression echo "" # Prepare new gravity database @@ -475,7 +475,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" + gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" fi echo "" done @@ -614,8 +614,8 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" cmd_ext="${2}" agent="${3}" adlistID="${4}" saveLocation="${5}" target="${6}" compression="${7}" - local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip + local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" + local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM listCurlBuffer=$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".phgpb") @@ -668,7 +668,7 @@ gravity_DownloadBlocklistFromUrl() { bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${PIHOLE_DNS_1} to download ${url}"; echo -ne " ${INFO} ${str} Pending..." - cmd_ext="--resolve $domain:$port:$ip $cmd_ext" + cmd_ext="--resolve $domain:$port:$ip" fi # shellcheck disable=SC2086 From d10d59303e48703cb167164b246b1deead4db984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 7 Apr 2023 21:44:49 +0200 Subject: [PATCH 039/462] There is no ! but an $ rule MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index f978cd0a..7c40bc0d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -779,7 +779,7 @@ gravity_ParseFileIntoDomains() { -e 's/\r$//' \ -e 's/\s*!.*//g' \ -e 's/\s*\[.*//g' \ - -e '/\#[!|?|@]{0,1}\#/d' \ + -e '/\#[$?@]{0,1}\#/d' \ -e 's/\s*#.*//g' \ -e 's/^.*\s+//g' \ -e '/^$/d' "${destination}" From dd3a7a4edb20edef094bc64e7348cfe769382140 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 8 Apr 2023 08:01:46 +0200 Subject: [PATCH 040/462] Only delete lines containing separator when preceded by a letter to reduce false positiv (deleting valid comments) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 7c40bc0d..925f4724 100755 --- a/gravity.sh +++ b/gravity.sh @@ -770,7 +770,7 @@ gravity_ParseFileIntoDomains() { # 2) Remove carriage returns # 3) Remove lines starting with ! (ABP Comments) # 4) Remove lines starting with [ (ABP Header) - # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") + # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") preceded by a letter # 6) Remove comments (text starting with "#", include possible spaces before the hash sign) # 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) # 8) Remove empty lines @@ -779,7 +779,7 @@ gravity_ParseFileIntoDomains() { -e 's/\r$//' \ -e 's/\s*!.*//g' \ -e 's/\s*\[.*//g' \ - -e '/\#[$?@]{0,1}\#/d' \ + -e '/[a-z]\#[$?@]{0,1}\#/d' \ -e 's/\s*#.*//g' \ -e 's/^.*\s+//g' \ -e '/^$/d' "${destination}" From 87a612f88436492c936903504196c37ebbfc0851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 15 Apr 2023 10:28:39 +0200 Subject: [PATCH 041/462] Trigger stale workflow on issue comments to remove stale label immediately MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 58a2e647..fa399be6 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -4,6 +4,7 @@ on: schedule: - cron: '0 8 * * *' workflow_dispatch: + issue_comment: jobs: stale: From c36d0257ec48d52237da08e22021a4ededa8177f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Apr 2023 10:57:39 +0000 Subject: [PATCH 042/462] Bump tox from 4.4.7 to 4.4.12 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.4.7 to 4.4.12. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.4.7...4.4.12) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 8ba6156f..be498400 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.2.2 pytest-xdist == 3.2.1 pytest-testinfra == 7.0.0 -tox == 4.4.7 +tox == 4.4.12 From d065afdbb19b1b1eeddbebe64738a4f8a5665048 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Apr 2023 10:57:41 +0000 Subject: [PATCH 043/462] Bump actions/checkout from 3.4.0 to 3.5.2 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.4.0 to 3.5.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.4.0...v3.5.2) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 01be8b25..f3bcf15a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 0fe850d1..fa7564a3 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.2 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 27867ef3..cd8a4030 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.2 - name: Check scripts in repository are executable run: | @@ -62,7 +62,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.2 - name: Set up Python 3.10 uses: actions/setup-python@v4.5.0 From 364fd38996656515a4fca1760e27eea8b946bf34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Apr 2023 12:34:19 +0000 Subject: [PATCH 044/462] Bump pytest from 7.2.2 to 7.3.1 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.2.2 to 7.3.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.2.2...7.3.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index be498400..8cd3ca77 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ docker-compose == 1.29.2 -pytest == 7.2.2 +pytest == 7.3.1 pytest-xdist == 3.2.1 pytest-testinfra == 7.0.0 tox == 4.4.12 From 5985d506f1e17bcc205d4b795a6635dc2d75c043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 16 Apr 2023 14:39:13 +0200 Subject: [PATCH 045/462] Run seperate job to trigger removal on comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index fa399be6..786940b2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -7,8 +7,8 @@ on: issue_comment: jobs: - stale: - + stale_action: + if: github.event_name != 'issue_comment' runs-on: ubuntu-latest permissions: issues: write @@ -25,3 +25,18 @@ jobs: exempt-all-issue-assignees: true operations-per-run: 300 close-issue-reason: 'not_planned' + + remove_stale: # trigger "stale" removal immediately when stale issues are commented on + if: github.event_name == 'issue_comment' + permissions: + contents: read # for actions/checkout + issues: write # to edit issues label + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3.4.0 + - name: Remove 'stale' label + run: gh issue edit ${{ github.event.issue.number }} --remove-label 'stale' + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + From 1a9dbec83caab696cd1f8e336b8b8ae4980dda21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 17 Apr 2023 20:52:51 +0200 Subject: [PATCH 046/462] Use env variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 786940b2..3d8a0763 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -6,6 +6,9 @@ on: workflow_dispatch: issue_comment: +env: + stale_label: stale + jobs: stale_action: if: github.event_name != 'issue_comment' @@ -20,7 +23,7 @@ jobs: days-before-stale: 30 days-before-close: 5 stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.' - stale-issue-label: 'stale' + stale-issue-label: $stale_label exempt-issue-labels: 'Internal, Fixed in next release, Bug: Confirmed, Documentation Needed' exempt-all-issue-assignees: true operations-per-run: 300 @@ -36,7 +39,7 @@ jobs: - name: Checkout uses: actions/checkout@v3.4.0 - name: Remove 'stale' label - run: gh issue edit ${{ github.event.issue.number }} --remove-label 'stale' + run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From eed4b70512fdd77a949fc7262809fd4390295e03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 19 Apr 2023 21:03:12 +0200 Subject: [PATCH 047/462] Add Fedora 38 to the test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 1 + test/_fedora_38.Dockerfile | 18 ++++++++++++++++++ test/tox.fedora_38.ini | 8 ++++++++ 3 files changed, 27 insertions(+) create mode 100644 test/_fedora_38.Dockerfile create mode 100644 test/tox.fedora_38.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cd8a4030..748f09b9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,6 +57,7 @@ jobs: centos_9, fedora_36, fedora_37, + fedora_38, ] env: DISTRO: ${{matrix.distro}} diff --git a/test/_fedora_38.Dockerfile b/test/_fedora_38.Dockerfile new file mode 100644 index 00000000..76f69771 --- /dev/null +++ b/test/_fedora_38.Dockerfile @@ -0,0 +1,18 @@ +FROM fedora:38 +RUN dnf install -y git initscripts + +ENV GITDIR /etc/.pihole +ENV SCRIPTDIR /opt/pihole + +RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole +ADD . $GITDIR +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR + +RUN true && \ + chmod +x $SCRIPTDIR/* + +ENV SKIP_INSTALL true +ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net + +#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.fedora_38.ini b/test/tox.fedora_38.ini new file mode 100644 index 00000000..0aa7612e --- /dev/null +++ b/test/tox.fedora_38.ini @@ -0,0 +1,8 @@ +[tox] +envlist = py3 + +[testenv] +allowlist_externals = docker +deps = -rrequirements.txt +commands = docker buildx build --load --progress plain -f _fedora_38.Dockerfile -t pytest_pihole:test_container ../ + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py From 0df06dc2fb5bb406ee75177d9bffed900f2cc12b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 Apr 2023 10:56:37 +0000 Subject: [PATCH 048/462] Bump actions/setup-python from 4.5.0 to 4.6.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.5.0 to 4.6.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.5.0...v4.6.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 748f09b9..2dceff1c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,7 +66,7 @@ jobs: uses: actions/checkout@v3.5.2 - name: Set up Python 3.10 - uses: actions/setup-python@v4.5.0 + uses: actions/setup-python@v4.6.0 with: python-version: "3.10" From 9bcb32356871b448b97a5fe6d6bd89fca72d3eca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 Apr 2023 13:37:35 +0000 Subject: [PATCH 049/462] Bump actions/checkout from 3.4.0 to 3.5.2 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.4.0 to 3.5.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.4.0...v3.5.2) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index cb49439b..fe28112c 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.4.0 + uses: actions/checkout@v3.5.2 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: From 83afff953f15075fdd37a1cd7f842a5bf026112d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 29 Apr 2023 10:57:31 +0000 Subject: [PATCH 050/462] Bump tox from 4.4.12 to 4.5.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.4.12 to 4.5.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.4.12...4.5.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 8cd3ca77..f13ae6ba 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.3.1 pytest-xdist == 3.2.1 pytest-testinfra == 7.0.0 -tox == 4.4.12 +tox == 4.5.1 From fd4e8766e45594b5e45ec121c272defe401a12d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 2 May 2023 22:44:35 +0200 Subject: [PATCH 051/462] Remove unused code from query.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/query.sh | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 12295fbc..1d3b0a29 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -69,31 +69,16 @@ if [[ -n "${str:-}" ]]; then exit 1 fi -# Scan an array of files for matching strings -scanList(){ - # Escape full stops - local domain="${1}" esc_domain="${1//./\\.}" lists="${2}" list_type="${3:-}" +# Scan a domain again a list of RegEX +scanRegExList(){ + local domain="${1}" list="${2}" - # Prevent grep from printing file path - cd "$piholeDir" || exit 1 + for entry in ${list}; do + if [[ "${domain}" =~ ${entry} ]]; then + printf "%b\n" "${entry}"; + fi + done - # Prevent grep -i matching slowly: https://bit.ly/2xFXtUX - export LC_CTYPE=C - - # /dev/null forces filename to be printed when only one list has been generated - case "${list_type}" in - "exact" ) grep -i -E -l "(^|(?/dev/null;; - # Iterate through each regexp and check whether it matches the domainQuery - # If it does, print the matching regexp and continue looping - # Input 1 - regexps | Input 2 - domainQuery - "regex" ) - for list in ${lists}; do - if [[ "${domain}" =~ ${list} ]]; then - printf "%b\n" "${list}"; - fi - done;; - * ) grep -i "${esc_domain}" "${lists}" /dev/null 2>/dev/null;; - esac } scanDatabaseTable() { @@ -188,7 +173,7 @@ scanRegexDatabaseTable() { # Split regexps over a new line str_regexList=$(printf '%s\n' "${regexList[@]}") # Check domain against regexps - mapfile -t regexMatches < <(scanList "${domain}" "${str_regexList}" "regex") + mapfile -t regexMatches < <(scanRegExList "${domain}" "${str_regexList}") # If there were regex matches if [[ "${#regexMatches[@]}" -ne 0 ]]; then # Split matching regexps over a new line From b74c6d5120d9630ff835bff70c531e5b16bf9e8d Mon Sep 17 00:00:00 2001 From: MichaIng Date: Sun, 5 Mar 2023 19:02:09 +0100 Subject: [PATCH 052/462] Add support for RISC-V 64-bit installs Signed-off-by: MichaIng --- automated install/basic-install.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 24fe9e54..1f3002e7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -357,7 +357,7 @@ package_manager_detect() { # These variable names match the ones for apt-get. See above for an explanation of what they are for. PKG_INSTALL=("${PKG_MANAGER}" install -y) # CentOS package manager returns 100 when there are packages to update so we need to || true to prevent the script from exiting. - PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src)' | wc -l || true" + PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates) PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat jq) @@ -2366,6 +2366,9 @@ get_binary_name() { # set the binary to be used l_binary="pihole-FTL-linux-x86_64" fi + elif [[ "${machine}" == "riscv64" ]]; then + printf "%b %b Detected riscv64 processor\\n" "${OVER}" "${TICK}" + l_binary="pihole-FTL-riscv64-linux-gnu" else # Something else - we try to use 32bit executable and warn the user if [[ ! "${machine}" == "i686" ]]; then From e6ae2e98cc3b6171c9777ed5bf935ba27c8f9d19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 7 May 2023 13:21:23 +0200 Subject: [PATCH 053/462] Don't source the install script in webpage.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 54 ++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 5ccdf733..6492a74d 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -22,12 +22,14 @@ readonly dnscustomcnamefile="/etc/dnsmasq.d/05-pihole-custom-cname.conf" readonly gravityDBfile="/etc/pihole/gravity.db" -# Source install script for ${setupVars}, ${PI_HOLE_BIN_DIR} and valid_ip() -readonly PI_HOLE_FILES_DIR="/etc/.pihole" -# shellcheck disable=SC2034 # used in basic-install to source the script without running it -SKIP_INSTALL="true" -source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" +readonly setupVars="/etc/pihole/setupVars.conf" +readonly PI_HOLE_BIN_DIR="/usr/local/bin" + +# Root of the web server +readonly webroot="/var/www/html" + +# Source utils script utilsfile="/opt/pihole/utils.sh" source "${utilsfile}" @@ -98,6 +100,47 @@ HashPassword() { echo "${return}" } +# Check an IP address to see if it is a valid one +valid_ip() { + # Local, named variables + local ip=${1} + local stat=1 + + # Regex matching one IPv4 component, i.e. an integer from 0 to 255. + # See https://tools.ietf.org/html/rfc1340 + local ipv4elem="(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?|0)"; + # Regex matching an optional port (starting with '#') range of 1-65536 + local portelem="(#(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?"; + # Build a full IPv4 regex from the above subexpressions + local regex="^${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}${portelem}$" + + # Evaluate the regex, and return the result + [[ $ip =~ ${regex} ]] + + stat=$? + return "${stat}" +} + +valid_ip6() { + local ip=${1} + local stat=1 + + # Regex matching one IPv6 element, i.e. a hex value from 0000 to FFFF + local ipv6elem="[0-9a-fA-F]{1,4}" + # Regex matching an IPv6 CIDR, i.e. 1 to 128 + local v6cidr="(\\/([1-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])){0,1}" + # Regex matching an optional port (starting with '#') range of 1-65536 + local portelem="(#(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?"; + # Build a full IPv6 regex from the above subexpressions + local regex="^(((${ipv6elem}))*((:${ipv6elem}))*::((${ipv6elem}))*((:${ipv6elem}))*|((${ipv6elem}))((:${ipv6elem})){7})${v6cidr}${portelem}$" + + # Evaluate the regex, and return the result + [[ ${ip} =~ ${regex} ]] + + stat=$? + return "${stat}" +} + SetWebPassword() { if [ "${SUDO_USER}" == "www-data" ]; then echo "Security measure: user www-data is not allowed to change webUI password!" @@ -613,7 +656,6 @@ Teleporter() { host="${host//./_}" filename="pi-hole-${host:-noname}-teleporter_${datetimestamp}.tar.gz" fi - # webroot is sourced from basic-install above php "${webroot}/admin/scripts/pi-hole/php/teleporter.php" > "${filename}" } From b8c3f6d999b3fec6375e09ecbf3430e258ad6294 Mon Sep 17 00:00:00 2001 From: MrDuck2742 Date: Fri, 5 May 2023 22:05:56 +0100 Subject: [PATCH 054/462] Adding Local DNS Records does not add to /etc/pihole/custom.list Fixes #5268 Signed-off-by: MrDuck2742 --- advanced/Scripts/webpage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 5ccdf733..5fa2475f 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -622,7 +622,7 @@ checkDomain() local domain validDomain # Convert to lowercase domain="${1,,}" - validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check + validDomain=$(grep -P "^((-|_)*[a-z0-9]((-|_)*[a-z0-9)*(-|_)*)(\\.(-|_)*([a-z0-9]((-|_)*[a-z0-9])*))*$" <<< "${domain}") # Valid chars check validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label echo "${validDomain}" } From a3e610dbf2e31afd4996641e2d0d77fcbcaee6d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 10 May 2023 06:52:51 +0200 Subject: [PATCH 055/462] Don't use '--suffix' in mktemp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index dea93267..3d624018 100755 --- a/gravity.sh +++ b/gravity.sh @@ -165,7 +165,10 @@ database_table_from_file() { src="${2}" backup_path="${piholeDir}/migration_backup" backup_file="${backup_path}/$(basename "${2}")" - tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".gravity")" + # Create a temporary file. We don't use '--suffix' here because not all + # implementations of mktemp support it, e.g. on Alpine + tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}")" + mv "${tmpFile}" "${tmpFile%.*}.gravity" local timestamp timestamp="$(date --utc +'%s')" @@ -438,7 +441,10 @@ gravity_DownloadBlocklists() { echo -e "${OVER} ${TICK} ${str}" fi - target="$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".gravity")" + # Create a temporary file. We don't use '--suffix' here because not all + # implementations of mktemp support it, e.g. on Alpine + target="$(mktemp -p "${GRAVITY_TMPDIR}")" + mv "${target}" "${target%.*}.gravity" # Use compression to reduce the amount of data that is transferred # between the Pi-hole and the ad list provider. Use this feature @@ -568,7 +574,9 @@ parseList() { fi # For completeness, we will get a count of non_domains (this is the number of entries left after stripping the source of comments/duplicates/false positives/domains) - invalid_domains="$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".ph-non-domains")" + # We don't use '--suffix' here because not all implementations of mktemp support it, e.g. on Alpine + invalid_domains=$(mktemp -p "${GRAVITY_TMPDIR}") + mv "${invalid_domains}" "${invalid_domains%.*}.ph-non-domains" num_non_domains=$(grep -Ev "^(${valid_domain_pattern}|${abp_domain_pattern}|${false_positives})$" "${src}" | tee "${invalid_domains}" | wc -l) @@ -618,7 +626,9 @@ gravity_DownloadBlocklistFromUrl() { local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM - listCurlBuffer=$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".phgpb") + # We don't use '--suffix' here because not all implementations of mktemp support it, e.g. on Alpine + listCurlBuffer="$(mktemp -p "${GRAVITY_TMPDIR}")" + mv "${listCurlBuffer}" "${listCurlBuffer%.*}.phgpb" # Determine if $saveLocation has read permission if [[ -r "${saveLocation}" && $url != "file"* ]]; then From eaded9fdb197f318d9fb72b110c0c38ce164a5fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 11 May 2023 22:16:21 +0200 Subject: [PATCH 056/462] Remove forgotten variables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 3d624018..1757aa6a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -544,7 +544,7 @@ gravity_DownloadBlocklists() { # is saved in gravtiy's info table to signal FTL if such domains are available abp_domains=0 parseList() { - local adlistID="${1}" src="${2}" target="${3}" temp_file temp_file_base non_domains sample_non_domains valid_domain_pattern abp_domain_pattern + local adlistID="${1}" src="${2}" target="${3}" valid_domain_pattern abp_domain_pattern # define valid domain patterns # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already From 8d91ca874be69840f07a89714be8e527de37c3a4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 13 May 2023 11:01:21 +0000 Subject: [PATCH 057/462] Bump pytest-xdist from 3.2.1 to 3.3.0 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.2.1 to 3.3.0. - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.2.1...v3.3.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index f13ae6ba..0965d7e6 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ docker-compose == 1.29.2 pytest == 7.3.1 -pytest-xdist == 3.2.1 +pytest-xdist == 3.3.0 pytest-testinfra == 7.0.0 tox == 4.5.1 From 73733308ba0f1c4e2cc5fb0c4aac6a9027349e48 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 15 May 2023 19:25:56 +0200 Subject: [PATCH 058/462] Use parseList function offered by pihole-FTL --- gravity.sh | 184 +++++++++-------------------------------------------- 1 file changed, 29 insertions(+), 155 deletions(-) diff --git a/gravity.sh b/gravity.sh index 1757aa6a..fe2097dd 100755 --- a/gravity.sh +++ b/gravity.sh @@ -129,7 +129,7 @@ gravity_swap_databases() { echo -e "${OVER} ${TICK} ${str}" if $oldAvail; then - echo -e " ${TICK} The old database remains available." + echo -e " ${TICK} The old database remains available" fi } @@ -145,18 +145,6 @@ update_gravity_timestamp() { return 0 } -# Update timestamp when the gravity table was last updated successfully -set_abp_info() { - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('abp_domains',${abp_domains});" - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to update ABP domain status in database ${gravityDBfile}\\n ${output}" - return 1 - fi - return 0 -} - # Import domains from file and store them in the specified database table database_table_from_file() { # Define locals @@ -239,17 +227,6 @@ database_table_from_file() { echo -e " ${CROSS} Unable to remove ${tmpFile}" } -# Update timestamp of last update of this list. We store this in the "old" database as all values in the new database will later be overwritten -database_adlist_updated() { - output=$( { printf ".timeout 30000\\nUPDATE adlist SET date_updated = (cast(strftime('%%s', 'now') as int)) WHERE id = %i;\\n" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to update timestamp of adlist with ID ${1} in database ${gravityDBfile}\\n ${output}" - gravity_Cleanup "error" - fi -} - # Check if a column with name ${2} exists in gravity table with name ${1} gravity_column_exists() { output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) @@ -267,7 +244,7 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_domains}" "${num_non_domains}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -441,10 +418,24 @@ gravity_DownloadBlocklists() { echo -e "${OVER} ${TICK} ${str}" fi - # Create a temporary file. We don't use '--suffix' here because not all - # implementations of mktemp support it, e.g. on Alpine - target="$(mktemp -p "${GRAVITY_TMPDIR}")" - mv "${target}" "${target%.*}.gravity" + str="Creating new gravity databases" + echo -ne " ${INFO} ${str}..." + + # Gravity copying SQL script + copyGravity="$(cat "${gravityDBcopy}")" + if [[ "${gravityDBfile}" != "${gravityDBfile_default}" ]]; then + # Replace default gravity script location by custom location + copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}" + fi + + output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" + return 1 + fi + echo -e "${OVER} ${TICK} ${str}" # Use compression to reduce the amount of data that is transferred # between the Pi-hole and the ad list provider. Use this feature @@ -486,116 +477,9 @@ gravity_DownloadBlocklists() { echo "" done - str="Creating new gravity databases" - echo -ne " ${INFO} ${str}..." - - # Gravity copying SQL script - copyGravity="$(cat "${gravityDBcopy}")" - if [[ "${gravityDBfile}" != "${gravityDBfile_default}" ]]; then - # Replace default gravity script location by custom location - copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}" - fi - - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to copy data from ${gravityDBfile} to ${gravityTEMPfile}\\n ${output}" - return 1 - fi - echo -e "${OVER} ${TICK} ${str}" - - str="Storing downloaded domains in new gravity database" - echo -ne " ${INFO} ${str}..." - output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" gravity\\n" "${target}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to fill gravity table in database ${gravityTEMPfile}\\n ${output}" - gravity_Cleanup "error" - else - echo -e "${OVER} ${TICK} ${str}" - fi - - if [[ "${status}" -eq 0 && -n "${output}" ]]; then - echo -e " Encountered non-critical SQL warnings. Please check the suitability of the lists you're using!\\n\\n SQL warnings:" - local warning file line lineno - while IFS= read -r line; do - echo " - ${line}" - warning="$(grep -oh "^[^:]*:[0-9]*" <<< "${line}")" - file="${warning%:*}" - lineno="${warning#*:}" - if [[ -n "${file}" && -n "${lineno}" ]]; then - echo -n " Line contains: " - awk "NR==${lineno}" < "${file}" - fi - done <<< "${output}" - echo "" - fi - - rm "${target}" > /dev/null 2>&1 || \ - echo -e " ${CROSS} Unable to remove ${target}" - gravity_Blackbody=true } - -# global variable to indicate if we found ABP style domains during the gravity run -# is saved in gravtiy's info table to signal FTL if such domains are available -abp_domains=0 -parseList() { - local adlistID="${1}" src="${2}" target="${3}" valid_domain_pattern abp_domain_pattern - - # define valid domain patterns - # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already - # adapted from https://stackoverflow.com/a/30007882 - - TLD_pattern="[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" - subdomain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)" - - valid_domain_pattern="${subdomain_pattern}+${TLD_pattern}" - - # supported ABP style: ||subdomain.domain.tld^ - # ${subdomain_pattern} is optional for ABP style, allowing TLD blocking: ||tld^ - abp_domain_pattern="\|\|${subdomain_pattern}*${TLD_pattern}\^" - - # A list of items of common local hostnames not to report as unusable - # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files - # but flagging them as unusable causes more confusion than it's worth - so we suppress them from the output - false_positives="localhost|localhost.localdomain|local|broadcasthost|localhost|ip6-localhost|ip6-loopback|lo0 localhost|ip6-localnet|ip6-mcastprefix|ip6-allnodes|ip6-allrouters|ip6-allhosts" - - # Extract valid domains from source file and append ,${adlistID} to each line and save count to variable for display. - num_domains=$(grep -E "^(${valid_domain_pattern}|${abp_domain_pattern})$" "${src}" | tee >(sed "s/$/,${adlistID}/" >> "${target}") | wc -l) - - # Check if the source file contained AdBlock Plus style domains, if so we set the global variable and inform the user - if grep -E "^${abp_domain_pattern}$" -m 1 -q "${src}"; then - echo " ${INFO} List contained AdBlock Plus style domains" - abp_domains=1 - fi - - # For completeness, we will get a count of non_domains (this is the number of entries left after stripping the source of comments/duplicates/false positives/domains) - # We don't use '--suffix' here because not all implementations of mktemp support it, e.g. on Alpine - invalid_domains=$(mktemp -p "${GRAVITY_TMPDIR}") - mv "${invalid_domains}" "${invalid_domains%.*}.ph-non-domains" - - num_non_domains=$(grep -Ev "^(${valid_domain_pattern}|${abp_domain_pattern}|${false_positives})$" "${src}" | tee "${invalid_domains}" | wc -l) - - # If there are unusable lines, we display some information about them. This is not error or major cause for concern. - if [[ "${num_non_domains}" -ne 0 ]]; then - type="domains" - if [[ "${abp_domains}" -ne 0 ]]; then - type="patterns" - fi - echo " ${INFO} Imported ${num_domains} ${type}, ignoring ${num_non_domains} non-domain entries" - echo " Sample of non-domain entries:" - invalid_lines=$(head -n 5 "${invalid_domains}") - echo "${invalid_lines}" | awk '{print " - " $0}' - else - echo " ${INFO} Imported ${num_domains} domains" - fi - rm "${invalid_domains}" -} - compareLists() { local adlistID="${1}" target="${2}" @@ -606,7 +490,6 @@ compareLists() { sha1sum "${target}" > "${target}.sha1" echo " ${INFO} List has been updated" database_adlist_status "${adlistID}" "1" - database_adlist_updated "${adlistID}" else echo " ${INFO} List stayed unchanged" database_adlist_status "${adlistID}" "2" @@ -616,7 +499,6 @@ compareLists() { sha1sum "${target}" > "${target}.sha1" # We assume here it was changed upstream database_adlist_status "${adlistID}" "1" - database_adlist_updated "${adlistID}" fi } @@ -716,9 +598,8 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table file - parseList "${adlistID}" "${saveLocation}" "${target}" + pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "2" - database_adlist_number "${adlistID}" done="true" # Check if $listCurlBuffer is a non-zero length file elif [[ -s "${listCurlBuffer}" ]]; then @@ -727,12 +608,9 @@ gravity_DownloadBlocklistFromUrl() { # Remove curl buffer file after its use rm "${listCurlBuffer}" # Add domains to database table file - parseList "${adlistID}" "${saveLocation}" "${target}" + pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" # Compare lists, are they identical? compareLists "${adlistID}" "${saveLocation}" - # Update gravity database table (status and updated timestamp are set in - # compareLists) - database_adlist_number "${adlistID}" done="true" else # Fall back to previously cached list if $listCurlBuffer is empty @@ -746,15 +624,12 @@ gravity_DownloadBlocklistFromUrl() { if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" # Add domains to database table file - parseList "${adlistID}" "${saveLocation}" "${target}" - database_adlist_number "${adlistID}" + pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" # Manually reset these two numbers because we do not call parseList here - num_domains=0 - num_non_domains=0 - database_adlist_number "${adlistID}" + database_adlist_number "${adlistID}" 0 0 database_adlist_status "${adlistID}" "4" fi fi @@ -797,9 +672,9 @@ gravity_Table_Count() { local str="${2}" local num num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")" - if [[ "${table}" == "vw_gravity" ]]; then + if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(DISTINCT domain) FROM ${table};")" + unique="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else @@ -809,7 +684,9 @@ gravity_Table_Count() { # Output count of blacklisted domains and regex filters gravity_ShowCount() { - gravity_Table_Count "vw_gravity" "gravity domains" "" + # Here we use the table "gravity" instead of the view "vw_gravity" for speed. + # It's safe to replace it here, because right after a gravity run both will show the exactly same number of domains. + gravity_Table_Count "gravity" "gravity domains" "" gravity_Table_Count "vw_blacklist" "exact blacklisted domains" gravity_Table_Count "vw_regex_blacklist" "regex blacklist filters" gravity_Table_Count "vw_whitelist" "exact whitelisted domains" @@ -1026,9 +903,6 @@ fi # Update gravity timestamp update_gravity_timestamp -# Set abp_domain info field -set_abp_info - # Ensure proper permissions are set for the database chown pihole:pihole "${gravityDBfile}" chmod g+w "${piholeDir}" "${gravityDBfile}" From f9b29cfb62bae81aa62e8cc76eee574d21bce9e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 May 2023 11:02:15 +0000 Subject: [PATCH 059/462] Bump pytest-xdist from 3.3.0 to 3.3.1 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.3.0 to 3.3.1. - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.3.0...v3.3.1) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0965d7e6..f2423ba4 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ docker-compose == 1.29.2 pytest == 7.3.1 -pytest-xdist == 3.3.0 +pytest-xdist == 3.3.1 pytest-testinfra == 7.0.0 tox == 4.5.1 From 10fe85933b4f0341587236b8ffa7894207689711 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 May 2023 13:41:46 +0000 Subject: [PATCH 060/462] Bump pytest-testinfra from 7.0.0 to 8.0.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 7.0.0 to 8.0.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/7.0.0...8.0.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index f2423ba4..55200286 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ docker-compose == 1.29.2 pytest == 7.3.1 pytest-xdist == 3.3.1 -pytest-testinfra == 7.0.0 +pytest-testinfra == 8.0.0 tox == 4.5.1 From c92826c1525c80cddca3d6fde7e2d8ce8a96bb46 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 May 2023 21:31:02 +0200 Subject: [PATCH 061/462] Do not copy info table during pihole -g Signed-off-by: DL6ER --- advanced/Templates/gravity_copy.sql | 2 -- 1 file changed, 2 deletions(-) diff --git a/advanced/Templates/gravity_copy.sql b/advanced/Templates/gravity_copy.sql index 3bea731d..ed11b61a 100644 --- a/advanced/Templates/gravity_copy.sql +++ b/advanced/Templates/gravity_copy.sql @@ -19,8 +19,6 @@ INSERT OR REPLACE INTO adlist SELECT * FROM OLD.adlist; DELETE FROM OLD.adlist_by_group WHERE adlist_id NOT IN (SELECT id FROM OLD.adlist); INSERT OR REPLACE INTO adlist_by_group SELECT * FROM OLD.adlist_by_group; -INSERT OR REPLACE INTO info SELECT * FROM OLD.info; - INSERT OR REPLACE INTO client SELECT * FROM OLD.client; DELETE FROM OLD.client_by_group WHERE client_id NOT IN (SELECT id FROM OLD.client); INSERT OR REPLACE INTO client_by_group SELECT * FROM OLD.client_by_group; From 5bdb089b7f542d953f91e4ee9c0accf891e07c82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 24 May 2023 22:54:47 +0200 Subject: [PATCH 062/462] Add Ubuntu 23 to test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 1 + test/_ubuntu_23.Dockerfile | 18 ++++++++++++++++++ test/tox.ubuntu_23.ini | 8 ++++++++ 3 files changed, 27 insertions(+) create mode 100644 test/_ubuntu_23.Dockerfile create mode 100644 test/tox.ubuntu_23.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2dceff1c..7d9f403e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,7 @@ jobs: debian_11, ubuntu_20, ubuntu_22, + ubuntu_23, centos_8, centos_9, fedora_36, diff --git a/test/_ubuntu_23.Dockerfile b/test/_ubuntu_23.Dockerfile new file mode 100644 index 00000000..f9b3910b --- /dev/null +++ b/test/_ubuntu_23.Dockerfile @@ -0,0 +1,18 @@ +FROM buildpack-deps:lunar-scm + +ENV GITDIR /etc/.pihole +ENV SCRIPTDIR /opt/pihole + +RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole +ADD . $GITDIR +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV DEBIAN_FRONTEND=noninteractive + +RUN true && \ + chmod +x $SCRIPTDIR/* + +ENV SKIP_INSTALL true +ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net + +#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.ubuntu_23.ini b/test/tox.ubuntu_23.ini new file mode 100644 index 00000000..767ed9ef --- /dev/null +++ b/test/tox.ubuntu_23.ini @@ -0,0 +1,8 @@ +[tox] +envlist = py3 + +[testenv:py3] +allowlist_externals = docker +deps = -rrequirements.txt +commands = docker buildx build --load --progress plain -f _ubuntu_23.Dockerfile -t pytest_pihole:test_container ../ + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py From 9f31ab8a6fd2dfe361c2383f4b59648eb9cd6e3f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 1 Jan 2023 12:34:40 +0000 Subject: [PATCH 063/462] Debug log does not need to check php/lighttpd Signed-off-by: Adam Warner --- advanced/Scripts/piholeDebug.sh | 63 +++++++-------------------------- 1 file changed, 12 insertions(+), 51 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index fa1cebbb..7462cba3 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -64,10 +64,8 @@ PIHOLE_SCRIPTS_DIRECTORY="/opt/pihole" BIN_DIRECTORY="/usr/local/bin" RUN_DIRECTORY="/run" LOG_DIRECTORY="/var/log/pihole" -WEB_SERVER_LOG_DIRECTORY="/var/log/lighttpd" -WEB_SERVER_CONFIG_DIRECTORY="/etc/lighttpd" -WEB_SERVER_CONFIG_DIRECTORY_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY}/conf.d" -WEB_SERVER_CONFIG_DIRECTORY_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY}/conf-enabled" +#WEB_SERVER_LOG_DIRECTORY="/var/log/lighttpd" #TODO: FTL access log? +#WEB_SERVER_CONFIG_DIRECTORY="/etc/lighttpd" #TODO: FTL access log? HTML_DIRECTORY="/var/www/html" WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin" SHM_DIRECTORY="/dev/shm" @@ -77,10 +75,8 @@ ETC="/etc" # https://discourse.pi-hole.net/t/what-files-does-pi-hole-use/1684 PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole" -WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" -WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" -WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN="${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}/15-pihole-admin.conf" -WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA="${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}/pihole-admin.conf" +#WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" +#WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" @@ -131,21 +127,19 @@ PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*" PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log" PIHOLE_FTL_LOG="$(get_ftl_conf_value "LOGFILE" "${LOG_DIRECTORY}/FTL.log")" -PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access-pihole.log" -PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error-pihole.log" +# PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access-pihole.log" #TODO: FTL Error log? +# PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error-pihole.log" #TODO: FTL Error log? RESOLVCONF="${ETC}/resolv.conf" DNSMASQ_CONF="${ETC}/dnsmasq.conf" # Store Pi-hole's processes in an array for easy use and parsing -PIHOLE_PROCESSES=( "lighttpd" "pihole-FTL" ) +PIHOLE_PROCESSES=( "pihole-FTL" ) # Store the required directories in an array so it can be parsed through REQUIRED_FILES=("${PIHOLE_CRON_FILE}" -"${WEB_SERVER_CONFIG_FILE}" -"${WEB_SERVER_CUSTOM_CONFIG_FILE}" -"${WEB_SERVER_PIHOLE_CONFIG_FILE_DEBIAN}" -"${WEB_SERVER_PIHOLE_CONFIG_FILE_FEDORA}" +# "${WEB_SERVER_CONFIG_FILE}" +# "${WEB_SERVER_CUSTOM_CONFIG_FILE}" "${PIHOLE_INSTALL_LOG_FILE}" "${PIHOLE_RAW_BLOCKLIST_FILES}" "${PIHOLE_LOCAL_HOSTS_FILE}" @@ -351,39 +345,6 @@ check_component_versions() { check_ftl_version } - -get_program_version() { - local program_name="${1}" - # Create a local variable so this function can be safely reused - local program_version - echo_current_diagnostic "${program_name} version" - # Evaluate the program we are checking, if it is any of the ones below, show the version - case "${program_name}" in - "lighttpd") program_version="$(${program_name} -v 2> /dev/null | head -n1 | cut -d '/' -f2 | cut -d ' ' -f1)" - ;; - "php") program_version="$(${program_name} -v 2> /dev/null | head -n1 | cut -d '-' -f1 | cut -d ' ' -f2)" - ;; - # If a match is not found, show an error - *) echo "Unrecognized program"; - esac - # If the program does not have a version (the variable is empty) - if [[ -z "${program_version}" ]]; then - # Display and error - log_write "${CROSS} ${COL_RED}${program_name} version could not be detected.${COL_NC}" - else - # Otherwise, display the version - log_write "${INFO} ${program_version}" - fi -} - -# These are the most critical dependencies of Pi-hole, so we check for them -# and their versions, using the functions above. -check_critical_program_versions() { - # Use the function created earlier and bundle them into one function that checks all the version numbers - get_program_version "lighttpd" - get_program_version "php" -} - os_check() { # This function gets a list of supported OS versions from a TXT record at versions.pi-hole.net # and determines whether or not the script is running on one of those systems @@ -737,10 +698,10 @@ compare_port_to_service_assigned() { check_required_ports() { echo_current_diagnostic "Ports in use" - # Since Pi-hole needs 53, 80, and 4711, check what they are being used by + # Since Pi-hole needs 53 and 4711, check what they are being used by # so we can detect any issues local resolver="pihole-FTL" - local web_server="lighttpd" + local web_server="pihole-FTL" local ftl="pihole-FTL" # Create an array for these ports in use ports_in_use=() @@ -1520,7 +1481,7 @@ initialize_debug # available to the other functions source_setup_variables check_component_versions -check_critical_program_versions +# check_critical_program_versions diagnose_operating_system check_selinux check_firewalld From 31ee15200d89ffd27fc16819ecebf3c360345993 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 1 Jan 2023 12:45:14 +0000 Subject: [PATCH 064/462] gut the install script of references to web server/lighttpd Signed-off-by: Adam Warner --- automated install/basic-install.sh | 233 ----------------------------- 1 file changed, 233 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 24fe9e54..3615804c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -58,8 +58,6 @@ EOM installLogLoc="/etc/pihole/install.log" # This is an important file as it contains information specific to the machine it's being installed on setupVars="/etc/pihole/setupVars.conf" -# Pi-hole uses lighttpd as a Web server, and this is the config file for it -lighttpdConfig="/etc/lighttpd/lighttpd.conf" # This is a file used for the colorized output coltable="/opt/pihole/COL_TABLE" @@ -110,13 +108,11 @@ c=70 # The runUnattended flag is one example of this reconfigure=false runUnattended=false -INSTALL_WEB_SERVER=true # Check arguments for the undocumented flags for var in "$@"; do case "$var" in "--reconfigure" ) reconfigure=true;; "--unattended" ) runUnattended=true;; - "--disable-install-webserver" ) INSTALL_WEB_SERVER=false;; esac done @@ -308,42 +304,12 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" # Update package cache update_package_cache || exit 1 - # Check for and determine version number (major and minor) of current php install - local phpVer="php" - if is_command php ; then - phpVer="$(php <<< "")" - # Check if the first character of the string is numeric - if [[ ${phpVer:0:1} =~ [1-9] ]]; then - printf " %b Existing PHP installation detected : PHP version %s\\n" "${INFO}" "${phpVer}" - printf -v phpInsMajor "%d" "$(php <<< "")" - printf -v phpInsMinor "%d" "$(php <<< "")" - phpVer="php$phpInsMajor.$phpInsMinor" - else - printf " %b No valid PHP installation detected!\\n" "${CROSS}" - printf " %b PHP version : %s\\n" "${INFO}" "${phpVer}" - printf " %b Aborting installation.\\n" "${CROSS}" - exit 1 - fi - fi # Packages required to perform the os_check (stored as an array) OS_CHECK_DEPS=(grep dnsutils) # Packages required to run this install script (stored as an array) INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole (stored as an array) PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) - # Packages required for the Web admin interface (stored as an array) - # It's useful to separate this from Pi-hole, since the two repos are also setup separately - PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") - # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole AdminLTE: https://www.php.net/manual/json.installation.php - if [[ -z "${phpInsMajor}" || "${phpInsMajor}" -lt 8 ]]; then - PIHOLE_WEB_DEPS+=("${phpVer}-json") - fi - # The Web server user, - LIGHTTPD_USER="www-data" - # group, - LIGHTTPD_GROUP="www-data" - # and config file - LIGHTTPD_CFG="lighttpd.conf.debian" # If apt-get is not found, check for rpm. elif is_command rpm ; then @@ -361,25 +327,6 @@ package_manager_detect() { OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates) PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat jq) - PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl) - LIGHTTPD_USER="lighttpd" - LIGHTTPD_GROUP="lighttpd" - LIGHTTPD_CFG="lighttpd.conf.fedora" - - # If the host OS is centos (or a derivative), epel is required for lighttpd - if ! grep -qiE 'fedora|fedberry' /etc/redhat-release; then - if rpm -qa | grep -qi 'epel'; then - printf " %b EPEL repository already installed\\n" "${TICK}" - else - local RH_RELEASE EPEL_PKG - # EPEL not already installed, add it based on the release version - RH_RELEASE=$(grep -oP '(?<= )[0-9]+(?=\.?)' /etc/redhat-release) - EPEL_PKG="https://dl.fedoraproject.org/pub/epel/epel-release-latest-${RH_RELEASE}.noarch.rpm" - printf " %b Enabling EPEL package repository (https://fedoraproject.org/wiki/EPEL)\\n" "${INFO}" - "${PKG_INSTALL[@]}" "${EPEL_PKG}" - printf " %b Installed %s\\n" "${TICK}" "${EPEL_PKG}" - fi - fi # If neither apt-get or yum/dnf package managers were found else @@ -1121,7 +1068,6 @@ setAdminFlag() { printf " %b Not installing Admin Web Interface\\n" "${INFO}" # Set the flag to not install the web interface INSTALL_WEB_INTERFACE=false - INSTALL_WEB_SERVER=false ;; "${DIALOG_ESC}") # User pressed @@ -1129,43 +1075,6 @@ setAdminFlag() { exit 1 ;; esac - - # If the user wants to install the Web admin interface (i.e. it has not been deselected above) and did not deselect the web server via command-line argument - if [[ "${INSTALL_WEB_INTERFACE}" == true && "${INSTALL_WEB_SERVER}" == true ]]; then - # Get list of required PHP modules, excluding base package (common) and handler (cgi) - local i php_modules - for i in "${PIHOLE_WEB_DEPS[@]}"; do [[ $i == 'php'* && $i != *'-common' && $i != *'-cgi' ]] && php_modules+=" ${i#*-}"; done - dialog --no-shadow --keep-tite \ - --backtitle "Pi-hole Installation" \ - --title "Web Server" \ - --yesno "\\n\\nA web server is required for the Admin Web Interface.\ -\\n\\nDo you want to install lighttpd and the required PHP modules?\ -\\n\\nNB: If you disable this, and, do not have an existing web server \ -and required PHP modules (${php_modules# }) installed, the web interface \ -will not function. Additionally the web server user needs to be member of \ -the \"pihole\" group for full functionality." \ - "${r}" "${c}" && result=0 || result=$? - - case ${result} in - "${DIALOG_OK}") - # If they chose yes, - printf " %b Installing lighttpd\\n" "${INFO}" - # Set the flag to install the web server - INSTALL_WEB_SERVER=true - ;; - "${DIALOG_CANCEL}") - # If they chose no, - printf " %b Not installing lighttpd\\n" "${INFO}" - # Set the flag to not install the web server - INSTALL_WEB_SERVER=false - ;; - "${DIALOG_ESC}") - # User pressed - printf " %b Escape pressed, exiting installer at web server choice.%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac - fi } # A function to display a list of example blocklists for users to select @@ -1397,79 +1306,6 @@ installConfigs() { fi install -T -m 0755 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL-prestart.sh" "${PI_HOLE_INSTALL_DIR}/pihole-FTL-prestart.sh" install -T -m 0755 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL-poststop.sh" "${PI_HOLE_INSTALL_DIR}/pihole-FTL-poststop.sh" - - # If the user chose to install the dashboard, - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # set permissions on /etc/lighttpd/lighttpd.conf so pihole user (other) can read the file - chmod o+x /etc/lighttpd - chmod o+r "${lighttpdConfig}" - - # Ensure /run/lighttpd exists and is owned by lighttpd user - # Needed for the php socket - mkdir -p /run/lighttpd - chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /run/lighttpd - - if grep -q -F "OVERWRITTEN BY PI-HOLE" "${lighttpdConfig}"; then - # Attempt to preserve backwards compatibility with older versions - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} "${lighttpdConfig}" - # Make the directories if they do not exist and set the owners - mkdir -p /var/cache/lighttpd/compress - chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /var/cache/lighttpd/compress - mkdir -p /var/cache/lighttpd/uploads - chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} /var/cache/lighttpd/uploads - fi - # Copy the config file to include for pihole admin interface - if [[ -d "/etc/lighttpd/conf.d" ]]; then - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/pihole-admin.conf /etc/lighttpd/conf.d/pihole-admin.conf - if grep -q -F 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' "${lighttpdConfig}"; then - : - else - echo 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' >> "${lighttpdConfig}" - fi - # Avoid some warnings trace from lighttpd, which might break tests - conf=/etc/lighttpd/conf.d/pihole-admin.conf - if lighttpd -f "${lighttpdConfig}" -tt 2>&1 | grep -q -F "WARNING: unknown config-key: dir-listing\."; then - echo '# Avoid some warnings trace from lighttpd, which might break tests' >> $conf - echo 'server.modules += ( "mod_dirlisting" )' >> $conf - fi - if lighttpd -f "${lighttpdConfig}" -tt 2>&1 | grep -q -F "warning: please use server.use-ipv6"; then - echo '# Avoid some warnings trace from lighttpd, which might break tests' >> $conf - echo 'server.use-ipv6 := "disable"' >> $conf - fi - elif [[ -d "/etc/lighttpd/conf-available" ]]; then - conf=/etc/lighttpd/conf-available/15-pihole-admin.conf - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/pihole-admin.conf $conf - - # Get the version number of lighttpd - version=$(dpkg-query -f='${Version}\n' --show lighttpd) - # Test if that version is greater than or euqal to 1.4.56 - if dpkg --compare-versions "$version" "ge" "1.4.56"; then - # If it is, then we don't need to disable the modules - # (server.modules duplication is ignored in lighttpd 1.4.56+) - : - else - # disable server.modules += ( ... ) in $conf to avoid module dups - if awk '!/^server\.modules/{print}' $conf > $conf.$$ && mv $conf.$$ $conf; then - : - else - rm $conf.$$ - fi - fi - - chmod 644 $conf - if is_command lighty-enable-mod ; then - lighty-enable-mod pihole-admin access accesslog redirect fastcgi setenv > /dev/null || true - else - # Otherwise, show info about installing them - printf " %b Warning: 'lighty-enable-mod' utility not found\\n" "${INFO}" - printf " Please ensure fastcgi is enabled if you experience issues\\n" - fi - else - # lighttpd config include dir not found - printf " %b Warning: lighttpd config include dir not found\\n" "${INFO}" - printf " Please manually install pihole-admin.conf\\n" - fi - fi } install_manpage() { @@ -1717,29 +1553,6 @@ install_dependent_packages() { return 0 } -# Install the Web interface dashboard -installPiholeWeb() { - # Install Sudoers file - local str="Installing sudoer file" - printf "\\n %b %s..." "${INFO}" "${str}" - # Make the .d directory if it doesn't exist, - install -d -m 755 /etc/sudoers.d/ - # and copy in the pihole sudoers file - install -m 0640 ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.sudo /etc/sudoers.d/pihole - # Add lighttpd user (OS dependent) to sudoers file - echo "${LIGHTTPD_USER} ALL=NOPASSWD: ${PI_HOLE_BIN_DIR}/pihole" >> /etc/sudoers.d/pihole - - # If the Web server user is lighttpd, - if [[ "$LIGHTTPD_USER" == "lighttpd" ]]; then - # Allow executing pihole via sudo with Fedora - # Usually /usr/local/bin ${PI_HOLE_BIN_DIR} is not permitted as directory for sudoable programs - echo "Defaults secure_path = /sbin:/bin:/usr/sbin:/usr/bin:${PI_HOLE_BIN_DIR}" >> /etc/sudoers.d/pihole - fi - # Set the strict permissions on the file - chmod 0440 /etc/sudoers.d/pihole - printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" -} - # Installs a cron file installCron() { # Install the cron job @@ -1838,9 +1651,7 @@ finalExports() { addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_1" "${PIHOLE_DNS_1}" addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_2" "${PIHOLE_DNS_2}" addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "${QUERY_LOGGING}" - addOrEditKeyValPair "${setupVars}" "INSTALL_WEB_SERVER" "${INSTALL_WEB_SERVER}" addOrEditKeyValPair "${setupVars}" "INSTALL_WEB_INTERFACE" "${INSTALL_WEB_INTERFACE}" - addOrEditKeyValPair "${setupVars}" "LIGHTTPD_ENABLED" "${LIGHTTPD_ENABLED}" addOrEditKeyValPair "${setupVars}" "CACHE_SIZE" "${CACHE_SIZE}" addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "${DNS_FQDN_REQUIRED:-true}" addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "${DNS_BOGUS_PRIV:-true}" @@ -1904,25 +1715,6 @@ installLogrotate() { # Install base files and web interface installPihole() { - # If the user wants to install the Web interface, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - if [[ ! -d "${webroot}" ]]; then - # make the Web directory if necessary - install -d -m 0755 ${webroot} - fi - - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # Set the owner and permissions - chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} ${webroot} - chmod 0775 ${webroot} - # Repair permissions if webroot is not world readable - chmod a+rx /var/www - chmod a+rx ${webroot} - # Give lighttpd access to the pihole group so the web interface can - # manage the gravity.db database - usermod -a -G pihole ${LIGHTTPD_USER} - fi - fi # Install base files and web interface if ! installScripts; then printf " %b Failure in dependent script copy function.\\n" "${CROSS}" @@ -2624,27 +2416,12 @@ main() { # Install the Core dependencies local dep_install_list=("${PIHOLE_DEPS[@]}") - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # And, if the setting says so, install the Web admin interface dependencies - dep_install_list+=("${PIHOLE_WEB_DEPS[@]}") - fi # Install packages used by the actual software printf " %b Checking for / installing Required dependencies for Pi-hole software...\\n" "${INFO}" install_dependent_packages "${dep_install_list[@]}" unset dep_install_list - # On some systems, lighttpd is not enabled on first install. We need to enable it here if the user - # has chosen to install the web interface, else the LIGHTTPD_ENABLED check will fail - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - enable_service lighttpd - fi - # Determine if lighttpd is correctly enabled - if check_service_active "lighttpd"; then - LIGHTTPD_ENABLED=true - else - LIGHTTPD_ENABLED=false - fi # Create the pihole user create_pihole_user @@ -2685,16 +2462,6 @@ main() { # but before starting or resarting the dnsmasq or ftl services disable_resolved_stublistener - # If the Web server was installed, - if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - if [[ "${LIGHTTPD_ENABLED}" == true ]]; then - restart_service lighttpd - enable_service lighttpd - else - printf " %b Lighttpd is disabled, skipping service restart\\n" "${INFO}" - fi - fi - printf " %b Restarting services...\\n" "${INFO}" # Start services From c520b293264f1761cd008e73f4ff3d5d22db8822 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 1 Jan 2023 12:47:00 +0000 Subject: [PATCH 065/462] some more lighttpd refs (tests can come later) Signed-off-by: Adam Warner --- advanced/Scripts/piholeDebug.sh | 57 ++++++++++----------- advanced/lighttpd.conf.debian | 73 --------------------------- advanced/lighttpd.conf.fedora | 87 --------------------------------- advanced/pihole-admin.conf | 82 ------------------------------- 4 files changed, 26 insertions(+), 273 deletions(-) delete mode 100644 advanced/lighttpd.conf.debian delete mode 100644 advanced/lighttpd.conf.fedora delete mode 100644 advanced/pihole-admin.conf diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 7462cba3..9f895aab 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -64,8 +64,6 @@ PIHOLE_SCRIPTS_DIRECTORY="/opt/pihole" BIN_DIRECTORY="/usr/local/bin" RUN_DIRECTORY="/run" LOG_DIRECTORY="/var/log/pihole" -#WEB_SERVER_LOG_DIRECTORY="/var/log/lighttpd" #TODO: FTL access log? -#WEB_SERVER_CONFIG_DIRECTORY="/etc/lighttpd" #TODO: FTL access log? HTML_DIRECTORY="/var/www/html" WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin" SHM_DIRECTORY="/dev/shm" @@ -75,9 +73,6 @@ ETC="/etc" # https://discourse.pi-hole.net/t/what-files-does-pi-hole-use/1684 PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole" -#WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" -#WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" - PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" @@ -762,34 +757,34 @@ check_networking() { [ -z "${DOCKER_VERSION}" ] && check_required_ports } -check_x_headers() { - # The X-Headers allow us to determine from the command line if the Web - # lighttpd.conf has a directive to show "X-Pi-hole: A black hole for Internet advertisements." - # in the header of any Pi-holed domain - # Similarly, it will show "X-Pi-hole: The Pi-hole Web interface is working!" if you view the header returned - # when accessing the dashboard (i.e curl -I pi.hole/admin/) - # server is operating correctly - echo_current_diagnostic "Dashboard headers" - # Use curl -I to get the header and parse out just the X-Pi-hole one - local full_curl_output_dashboard - local dashboard - full_curl_output_dashboard="$(curl -Is localhost/admin/)" - dashboard=$(echo "${full_curl_output_dashboard}" | awk '/X-Pi-hole/' | tr -d '\r') - # Store what the X-Header should be in variables for comparison later - local dashboard_working - dashboard_working="X-Pi-hole: The Pi-hole Web interface is working!" +# check_x_headers() { +# # The X-Headers allow us to determine from the command line if the Web +# # lighttpd.conf has a directive to show "X-Pi-hole: A black hole for Internet advertisements." +# # in the header of any Pi-holed domain +# # Similarly, it will show "X-Pi-hole: The Pi-hole Web interface is working!" if you view the header returned +# # when accessing the dashboard (i.e curl -I pi.hole/admin/) +# # server is operating correctly +# echo_current_diagnostic "Dashboard headers" +# # Use curl -I to get the header and parse out just the X-Pi-hole one +# local full_curl_output_dashboard +# local dashboard +# full_curl_output_dashboard="$(curl -Is localhost/admin/)" +# dashboard=$(echo "${full_curl_output_dashboard}" | awk '/X-Pi-hole/' | tr -d '\r') +# # Store what the X-Header should be in variables for comparison later +# local dashboard_working +# dashboard_working="X-Pi-hole: The Pi-hole Web interface is working!" - # If the X-Header matches what a working system should have, - if [[ $dashboard == "$dashboard_working" ]]; then - # then we can show a success - log_write "$TICK Web interface X-Header: ${COL_GREEN}${dashboard}${COL_NC}" - else - # Otherwise, it's a failure since the X-Headers either don't exist or have been modified in some way - log_write "$CROSS Web interface X-Header: ${COL_RED}X-Header does not match or could not be retrieved.${COL_NC}" +# # If the X-Header matches what a working system should have, +# if [[ $dashboard == "$dashboard_working" ]]; then +# # then we can show a success +# log_write "$TICK Web interface X-Header: ${COL_GREEN}${dashboard}${COL_NC}" +# else +# # Otherwise, it's a failure since the X-Headers either don't exist or have been modified in some way +# log_write "$CROSS Web interface X-Header: ${COL_RED}X-Header does not match or could not be retrieved.${COL_NC}" - log_write "${COL_RED}${full_curl_output_dashboard}${COL_NC}" - fi -} +# log_write "${COL_RED}${full_curl_output_dashboard}${COL_NC}" +# fi +# } dig_at() { # We need to test if Pi-hole can properly resolve domain names diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian deleted file mode 100644 index f31f7bcd..00000000 --- a/advanced/lighttpd.conf.debian +++ /dev/null @@ -1,73 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Lighttpd config for Pi-hole -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -################################################################################################### -# IF THIS HEADER EXISTS, THE FILE WILL BE OVERWRITTEN BY PI-HOLE'S UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE WILL BE LOST ON THE NEXT UPDATE UNLESS YOU REMOVE THIS HEADER # -# # -# ENSURE THAT YOU DO NOT REMOVE THE REQUIRED LINE: # -# # -# include "/etc/lighttpd/conf-enabled/*.conf" # -# # -################################################################################################### - -server.modules = ( - "mod_access", - "mod_auth", - "mod_expire", - "mod_redirect", - "mod_setenv", - "mod_rewrite" -) - -server.document-root = "/var/www/html" -server.upload-dirs = ( "/var/cache/lighttpd/uploads" ) -server.errorlog = "/var/log/lighttpd/error-pihole.log" -server.pid-file = "/run/lighttpd.pid" -server.username = "www-data" -server.groupname = "www-data" -# For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator -# e.g. server.port := 8000 -server.port = 80 - -# Allow streaming response -# reference: https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_stream-response-bodyDetails -server.stream-response-body = 1 -#ssl.read-ahead = "disable" - -index-file.names = ( "index.php", "index.html", "index.lighttpd.html" ) -url.access-deny = ( "~", ".inc", ".md", ".yml", ".ini" ) -static-file.exclude-extensions = ( ".php", ".pl", ".fcgi" ) - -mimetype.assign = ( - ".ico" => "image/x-icon", - ".jpeg" => "image/jpeg", - ".jpg" => "image/jpeg", - ".png" => "image/png", - ".svg" => "image/svg+xml", - ".css" => "text/css; charset=utf-8", - ".html" => "text/html; charset=utf-8", - ".js" => "text/javascript; charset=utf-8", - ".json" => "application/json; charset=utf-8", - ".map" => "application/json; charset=utf-8", - ".txt" => "text/plain; charset=utf-8", - ".eot" => "application/vnd.ms-fontobject", - ".otf" => "font/otf", - ".ttc" => "font/collection", - ".ttf" => "font/ttf", - ".woff" => "font/woff", - ".woff2" => "font/woff2" -) - -# Add user chosen options held in (optional) external file -include "external*.conf" - -# default listening port for IPv6 falls back to the IPv4 port -include_shell "/usr/share/lighttpd/use-ipv6.pl " + server.port -include "/etc/lighttpd/conf-enabled/*.conf" diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora deleted file mode 100644 index e09d7760..00000000 --- a/advanced/lighttpd.conf.fedora +++ /dev/null @@ -1,87 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Lighttpd config for Pi-hole -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -################################################################################################### -# IF THIS HEADER EXISTS, THE FILE WILL BE OVERWRITTEN BY PI-HOLE'S UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE WILL BE LOST ON THE NEXT UPDATE UNLESS YOU REMOVE THIS HEADER # -# # -# ENSURE THAT YOU DO NOT REMOVE THE REQUIRED LINE: # -# # -# include "/etc/lighttpd/conf.d/pihole-admin.conf" # -# # -################################################################################################### - -server.modules = ( - "mod_access", - "mod_auth", - "mod_expire", - "mod_fastcgi", - "mod_accesslog", - "mod_redirect", - "mod_setenv", - "mod_rewrite" -) - -server.document-root = "/var/www/html" -server.upload-dirs = ( "/var/cache/lighttpd/uploads" ) -server.errorlog = "/var/log/lighttpd/error-pihole.log" -server.pid-file = "/run/lighttpd.pid" -server.username = "lighttpd" -server.groupname = "lighttpd" -# For lighttpd version 1.4.46 or above, the port can be overwritten in `/etc/lighttpd/external.conf` using the := operator -# e.g. server.port := 8000 -server.port = 80 - -# Allow streaming response -# reference: https://redmine.lighttpd.net/projects/lighttpd/wiki/Server_stream-response-bodyDetails -server.stream-response-body = 1 -#ssl.read-ahead = "disable" - -index-file.names = ( "index.php", "index.html", "index.lighttpd.html" ) -url.access-deny = ( "~", ".inc", ".md", ".yml", ".ini" ) -static-file.exclude-extensions = ( ".php", ".pl", ".fcgi" ) - -mimetype.assign = ( - ".ico" => "image/x-icon", - ".jpeg" => "image/jpeg", - ".jpg" => "image/jpeg", - ".png" => "image/png", - ".svg" => "image/svg+xml", - ".css" => "text/css; charset=utf-8", - ".html" => "text/html; charset=utf-8", - ".js" => "text/javascript; charset=utf-8", - ".json" => "application/json; charset=utf-8", - ".map" => "application/json; charset=utf-8", - ".txt" => "text/plain; charset=utf-8", - ".eot" => "application/vnd.ms-fontobject", - ".otf" => "font/otf", - ".ttc" => "font/collection", - ".ttf" => "font/ttf", - ".woff" => "font/woff", - ".woff2" => "font/woff2" -) - -# Add user chosen options held in (optional) external file -include "external*.conf" - -# default listening port for IPv6 falls back to the IPv4 port -#include_shell "/usr/share/lighttpd/use-ipv6.pl " + server.port -#include_shell "/usr/share/lighttpd/create-mime.assign.pl" -#include_shell "/usr/share/lighttpd/include-conf-enabled.pl" - -fastcgi.server = ( - ".php" => ( - "localhost" => ( - "socket" => "/tmp/php-fastcgi.socket", - "bin-path" => "/usr/bin/php-cgi" - ) - ) -) - -include "/etc/lighttpd/conf.d/pihole-admin.conf" diff --git a/advanced/pihole-admin.conf b/advanced/pihole-admin.conf deleted file mode 100644 index 0bb6eac9..00000000 --- a/advanced/pihole-admin.conf +++ /dev/null @@ -1,82 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Lighttpd config for Pi-hole -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -############################################################################### -# FILE AUTOMATICALLY OVERWRITTEN BY PI-HOLE INSTALL/UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE # -############################################################################### - -server.errorlog := "/var/log/lighttpd/error-pihole.log" - -$HTTP["url"] =~ "^/admin/" { - server.document-root = "/var/www/html" - server.stream-response-body = 1 - accesslog.filename = "/var/log/lighttpd/access-pihole.log" - accesslog.format = "%{%s}t|%h|%V|%r|%s|%b" - - fastcgi.server = ( - ".php" => ( - "localhost" => ( - "socket" => "/run/lighttpd/pihole-php-fastcgi.socket", - "bin-path" => "/usr/bin/php-cgi", - "min-procs" => 1, - "max-procs" => 1, - "bin-environment" => ( - "PHP_FCGI_CHILDREN" => "4", - "PHP_FCGI_MAX_REQUESTS" => "10000", - ), - "bin-copy-environment" => ( - "PATH", "SHELL", "USER" - ), - "broken-scriptfilename" => "enable", - ) - ) - ) - - # X-Pi-hole is a response header for debugging using curl -I - # X-Frame-Options prevents clickjacking attacks and helps ensure your content is not embedded into other sites via < frame >, < iframe > or < object >. - # X-XSS-Protection sets the configuration for the cross-site scripting filters built into most browsers. This is important because it tells the browser to block the response if a malicious script has been inserted from a user input. (deprecated; disabled) - # X-Content-Type-Options stops a browser from trying to MIME-sniff the content type and forces it to stick with the declared content-type. This is important because the browser will only load external resources if their content-type matches what is expected, and not malicious hidden code. - # Content-Security-Policy tells the browser where resources are allowed to be loaded and if it’s allowed to parse/run inline styles or Javascript. This is important because it prevents content injection attacks, such as Cross Site Scripting (XSS). - # X-Permitted-Cross-Domain-Policies is an XML document that grants a web client, such as Adobe Flash Player or Adobe Acrobat (though not necessarily limited to these), permission to handle data across domains. - # Referrer-Policy allows control/restriction of the amount of information present in the referral header for links away from your page—the URL path or even if the header is sent at all. - setenv.add-response-header = ( - "X-Pi-hole" => "The Pi-hole Web interface is working!", - "X-Frame-Options" => "DENY", - "X-XSS-Protection" => "0", - "X-Content-Type-Options" => "nosniff", - "Content-Security-Policy" => "default-src 'self' 'unsafe-inline';", - "X-Permitted-Cross-Domain-Policies" => "none", - "Referrer-Policy" => "same-origin" - ) - - # Block . files from being served, such as .git, .github, .gitignore - $HTTP["url"] =~ "^/admin/\." { - url.access-deny = ("") - } - - # allow teleporter and API qr code iframe on settings page - $HTTP["url"] =~ "/(teleporter|api_token)\.php$" { - $HTTP["referer"] =~ "/admin/settings\.php" { - setenv.set-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) - } - } -} -else $HTTP["url"] == "/admin" { - url.redirect = ("" => "/admin/") -} - -$HTTP["host"] == "pi.hole" { - $HTTP["url"] == "/" { - url.redirect = ("" => "/admin/") - } -} - -# (keep this on one line for basic-install.sh filtering during install) -server.modules += ( "mod_access", "mod_accesslog", "mod_redirect", "mod_fastcgi", "mod_setenv" ) From d2d11959289fedfc0490921269a35e864742cf82 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 1 Jan 2023 12:49:07 +0000 Subject: [PATCH 066/462] Needs tweaking, offer chance to change web interface port Signed-off-by: Adam Warner --- automated install/basic-install.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 3615804c..d9e22e08 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -92,6 +92,7 @@ IPV6_ADDRESS=${IPV6_ADDRESS} # Give settings their default values. These may be changed by prompts later in the script. QUERY_LOGGING=true INSTALL_WEB_INTERFACE=true +WEBPORT=8080 PRIVACY_LEVEL=0 CACHE_SIZE=10000 @@ -1062,6 +1063,14 @@ setAdminFlag() { printf " %b Installing Admin Web Interface\\n" "${INFO}" # Set the flag to install the web interface INSTALL_WEB_INTERFACE=true + + # Web port TODO: Below whiptail copy pasted from a previous go at this. needs converting to dialog + # Ask for the IPv4 address + WEBPORT=$(whiptail --backtitle "Setting web interface port" --title "Web Port" --inputbox "By default, pihole-FTL listens for http traffic on port 8080. If you wish to change the port, you may do so now. You can also do it later by editing /etc/pihole/pihole-FTL.conf" "${r}" "${c}" "${WEBPORT}" 3>&1 1>&2 2>&3) || \ + # Canceling IPv4 settings window + { echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } + printf " %b The Web interface will be accessible on port: %s\\n" "${INFO}" "${WEBPORT}" + ;; "${DIALOG_CANCEL}") # If they chose no, @@ -1662,6 +1671,9 @@ finalExports() { # Set the privacy level addOrEditKeyValPair "${FTL_CONFIG_FILE}" "PRIVACYLEVEL" "${PRIVACY_LEVEL}" + # Set the web port + addOrEditKeyValPair "${FTL_CONFIG_FILE}" "WEBPORT" "${PRIVACY_LEVEL}" + # Bring in the current settings and the functions to manipulate them source "${setupVars}" # shellcheck source=advanced/Scripts/webpage.sh @@ -1819,7 +1831,7 @@ displayFinalMessage() { # If the user wants to install the dashboard, if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then # Store a message in a variable and display it - additional="View the web interface at http://pi.hole/admin or http://${IPV4_ADDRESS%/*}/admin\\n\\nYour Admin Webpage login password is ${pwstring}" + additional="View the web interface at http://pi.hole/admin:${WEBPORT} or http://${IPV4_ADDRESS%/*}:${WEBPORT}/admin\\n\\nYour Admin Webpage login password is ${pwstring}" fi # Final completion message to user @@ -2524,7 +2536,7 @@ main() { if [[ "${useUpdateVars}" == false ]]; then # If the Web interface was installed, if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - printf " %b View the web interface at http://pi.hole/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}" + printf " %b View the web interface at http://pi.hole:${WEBPORT}/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}:${WEBPORT}" fi # Explain to the user how to use Pi-hole as their DNS server printf " %b You may now configure your devices to use the Pi-hole as their DNS server\\n" "${INFO}" From 31f16510e2e469b48780625ed45d0f4312f9a9da Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 13 Jan 2023 21:23:50 +0000 Subject: [PATCH 067/462] Clear out some no longer needed ftl/dnsmasq config code Signed-off-by: Adam Warner --- advanced/01-pihole.conf | 35 -- advanced/06-rfc6761.conf | 42 -- advanced/dnsmasq.conf.original | 648 ----------------------------- automated install/basic-install.sh | 114 +---- 4 files changed, 20 insertions(+), 819 deletions(-) delete mode 100644 advanced/01-pihole.conf delete mode 100644 advanced/06-rfc6761.conf delete mode 100644 advanced/dnsmasq.conf.original diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf deleted file mode 100644 index 677910f6..00000000 --- a/advanced/01-pihole.conf +++ /dev/null @@ -1,35 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Dnsmasq config for Pi-hole's FTLDNS -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -############################################################################### -# FILE AUTOMATICALLY POPULATED BY PI-HOLE INSTALL/UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE # -# # -# IF YOU WISH TO CHANGE THE UPSTREAM SERVERS, CHANGE THEM IN: # -# /etc/pihole/setupVars.conf # -# # -# ANY OTHER CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE # -# WITHIN /etc/dnsmasq.d/yourname.conf # -############################################################################### - -addn-hosts=/etc/pihole/local.list -addn-hosts=/etc/pihole/custom.list - -domain-needed - -localise-queries - -bogus-priv - -no-resolv - -log-queries -log-facility=/var/log/pihole/pihole.log - -log-async diff --git a/advanced/06-rfc6761.conf b/advanced/06-rfc6761.conf deleted file mode 100644 index fcdd0010..00000000 --- a/advanced/06-rfc6761.conf +++ /dev/null @@ -1,42 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2021 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# RFC 6761 config file for Pi-hole -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -############################################################################### -# FILE AUTOMATICALLY POPULATED BY PI-HOLE INSTALL/UPDATE PROCEDURE. # -# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE # -# # -# CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE # -# WITHIN /etc/dnsmasq.d/yourname.conf # -############################################################################### - -# RFC 6761: Caching DNS servers SHOULD recognize -# test, localhost, invalid -# names as special and SHOULD NOT attempt to look up NS records for them, or -# otherwise query authoritative DNS servers in an attempt to resolve these -# names. -server=/test/ -server=/localhost/ -server=/invalid/ - -# The same RFC requests something similar for -# 10.in-addr.arpa. 21.172.in-addr.arpa. 27.172.in-addr.arpa. -# 16.172.in-addr.arpa. 22.172.in-addr.arpa. 28.172.in-addr.arpa. -# 17.172.in-addr.arpa. 23.172.in-addr.arpa. 29.172.in-addr.arpa. -# 18.172.in-addr.arpa. 24.172.in-addr.arpa. 30.172.in-addr.arpa. -# 19.172.in-addr.arpa. 25.172.in-addr.arpa. 31.172.in-addr.arpa. -# 20.172.in-addr.arpa. 26.172.in-addr.arpa. 168.192.in-addr.arpa. -# Pi-hole implements this via the dnsmasq option "bogus-priv" (see -# 01-pihole.conf) because this also covers IPv6. - -# OpenWRT furthermore blocks bind, local, onion domains -# see https://git.openwrt.org/?p=openwrt/openwrt.git;a=blob_plain;f=package/network/services/dnsmasq/files/rfc6761.conf;hb=HEAD -# and https://www.iana.org/assignments/special-use-domain-names/special-use-domain-names.xhtml -# We do not include the ".local" rule ourselves, see https://github.com/pi-hole/pi-hole/pull/4282#discussion_r689112972 -server=/bind/ -server=/onion/ diff --git a/advanced/dnsmasq.conf.original b/advanced/dnsmasq.conf.original deleted file mode 100644 index 4aa5a8bf..00000000 --- a/advanced/dnsmasq.conf.original +++ /dev/null @@ -1,648 +0,0 @@ -# Configuration file for dnsmasq. -# -# Format is one option per line, legal options are the same -# as the long options legal on the command line. See -# "/usr/sbin/dnsmasq --help" or "man 8 dnsmasq" for details. - -# Listen on this specific port instead of the standard DNS port -# (53). Setting this to zero completely disables DNS function, -# leaving only DHCP and/or TFTP. -#port=5353 - -# The following two options make you a better netizen, since they -# tell dnsmasq to filter out queries which the public DNS cannot -# answer, and which load the servers (especially the root servers) -# unnecessarily. If you have a dial-on-demand link they also stop -# these requests from bringing up the link unnecessarily. - -# Never forward plain names (without a dot or domain part) -#domain-needed -# Never forward addresses in the non-routed address spaces. -#bogus-priv - -# Uncomment these to enable DNSSEC validation and caching: -# (Requires dnsmasq to be built with DNSSEC option.) -#conf-file=%%PREFIX%%/share/dnsmasq/trust-anchors.conf -#dnssec - -# Replies which are not DNSSEC signed may be legitimate, because the domain -# is unsigned, or may be forgeries. Setting this option tells dnsmasq to -# check that an unsigned reply is OK, by finding a secure proof that a DS -# record somewhere between the root and the domain does not exist. -# The cost of setting this is that even queries in unsigned domains will need -# one or more extra DNS queries to verify. -#dnssec-check-unsigned - -# Uncomment this to filter useless windows-originated DNS requests -# which can trigger dial-on-demand links needlessly. -# Note that (amongst other things) this blocks all SRV requests, -# so don't use it if you use eg Kerberos, SIP, XMMP or Google-talk. -# This option only affects forwarding, SRV records originating for -# dnsmasq (via srv-host= lines) are not suppressed by it. -#filterwin2k - -# Change this line if you want dns to get its upstream servers from -# somewhere other that /etc/resolv.conf -#resolv-file= - -# By default, dnsmasq will send queries to any of the upstream -# servers it knows about and tries to favor servers to are known -# to be up. Uncommenting this forces dnsmasq to try each query -# with each server strictly in the order they appear in -# /etc/resolv.conf -#strict-order - -# If you don't want dnsmasq to read /etc/resolv.conf or any other -# file, getting its servers from this file instead (see below), then -# uncomment this. -#no-resolv - -# If you don't want dnsmasq to poll /etc/resolv.conf or other resolv -# files for changes and re-read them then uncomment this. -#no-poll - -# Add other name servers here, with domain specs if they are for -# non-public domains. -#server=/localnet/192.168.0.1 - -# Example of routing PTR queries to nameservers: this will send all -# address->name queries for 192.168.3/24 to nameserver 10.1.2.3 -#server=/3.168.192.in-addr.arpa/10.1.2.3 - -# Add local-only domains here, queries in these domains are answered -# from /etc/hosts or DHCP only. -#local=/localnet/ - -# Add domains which you want to force to an IP address here. -# The example below send any host in double-click.net to a local -# web-server. -#address=/double-click.net/127.0.0.1 - -# --address (and --server) work with IPv6 addresses too. -#address=/www.thekelleys.org.uk/fe80::20d:60ff:fe36:f83 - -# Add the IPs of all queries to yahoo.com, google.com, and their -# subdomains to the vpn and search ipsets: -#ipset=/yahoo.com/google.com/vpn,search - -# You can control how dnsmasq talks to a server: this forces -# queries to 10.1.2.3 to be routed via eth1 -# server=10.1.2.3@eth1 - -# and this sets the source (ie local) address used to talk to -# 10.1.2.3 to 192.168.1.1 port 55 (there must be a interface with that -# IP on the machine, obviously). -# server=10.1.2.3@192.168.1.1#55 - -# If you want dnsmasq to change uid and gid to something other -# than the default, edit the following lines. -#user= -#group= - -# If you want dnsmasq to listen for DHCP and DNS requests only on -# specified interfaces (and the loopback) give the name of the -# interface (eg eth0) here. -# Repeat the line for more than one interface. -#interface= -# Or you can specify which interface _not_ to listen on -#except-interface= -# Or which to listen on by address (remember to include 127.0.0.1 if -# you use this.) -#listen-address= -# If you want dnsmasq to provide only DNS service on an interface, -# configure it as shown above, and then use the following line to -# disable DHCP and TFTP on it. -#no-dhcp-interface= - -# On systems which support it, dnsmasq binds the wildcard address, -# even when it is listening on only some interfaces. It then discards -# requests that it shouldn't reply to. This has the advantage of -# working even when interfaces come and go and change address. If you -# want dnsmasq to really bind only the interfaces it is listening on, -# uncomment this option. About the only time you may need this is when -# running another nameserver on the same machine. -#bind-interfaces - -# If you don't want dnsmasq to read /etc/hosts, uncomment the -# following line. -#no-hosts -# or if you want it to read another file, as well as /etc/hosts, use -# this. -#addn-hosts=/etc/banner_add_hosts - -# Set this (and domain: see below) if you want to have a domain -# automatically added to simple names in a hosts-file. -#expand-hosts - -# Set the domain for dnsmasq. this is optional, but if it is set, it -# does the following things. -# 1) Allows DHCP hosts to have fully qualified domain names, as long -# as the domain part matches this setting. -# 2) Sets the "domain" DHCP option thereby potentially setting the -# domain of all systems configured by DHCP -# 3) Provides the domain part for "expand-hosts" -#domain=thekelleys.org.uk - -# Set a different domain for a particular subnet -#domain=wireless.thekelleys.org.uk,192.168.2.0/24 - -# Same idea, but range rather then subnet -#domain=reserved.thekelleys.org.uk,192.68.3.100,192.168.3.200 - -# Uncomment this to enable the integrated DHCP server, you need -# to supply the range of addresses available for lease and optionally -# a lease time. If you have more than one network, you will need to -# repeat this for each network on which you want to supply DHCP -# service. -#dhcp-range=192.168.0.50,192.168.0.150,12h - -# This is an example of a DHCP range where the netmask is given. This -# is needed for networks we reach the dnsmasq DHCP server via a relay -# agent. If you don't know what a DHCP relay agent is, you probably -# don't need to worry about this. -#dhcp-range=192.168.0.50,192.168.0.150,255.255.255.0,12h - -# This is an example of a DHCP range which sets a tag, so that -# some DHCP options may be set only for this network. -#dhcp-range=set:red,192.168.0.50,192.168.0.150 - -# Use this DHCP range only when the tag "green" is set. -#dhcp-range=tag:green,192.168.0.50,192.168.0.150,12h - -# Specify a subnet which can't be used for dynamic address allocation, -# is available for hosts with matching --dhcp-host lines. Note that -# dhcp-host declarations will be ignored unless there is a dhcp-range -# of some type for the subnet in question. -# In this case the netmask is implied (it comes from the network -# configuration on the machine running dnsmasq) it is possible to give -# an explicit netmask instead. -#dhcp-range=192.168.0.0,static - -# Enable DHCPv6. Note that the prefix-length does not need to be specified -# and defaults to 64 if missing/ -#dhcp-range=1234::2, 1234::500, 64, 12h - -# Do Router Advertisements, BUT NOT DHCP for this subnet. -#dhcp-range=1234::, ra-only - -# Do Router Advertisements, BUT NOT DHCP for this subnet, also try and -# add names to the DNS for the IPv6 address of SLAAC-configured dual-stack -# hosts. Use the DHCPv4 lease to derive the name, network segment and -# MAC address and assume that the host will also have an -# IPv6 address calculated using the SLAAC algorithm. -#dhcp-range=1234::, ra-names - -# Do Router Advertisements, BUT NOT DHCP for this subnet. -# Set the lifetime to 46 hours. (Note: minimum lifetime is 2 hours.) -#dhcp-range=1234::, ra-only, 48h - -# Do DHCP and Router Advertisements for this subnet. Set the A bit in the RA -# so that clients can use SLAAC addresses as well as DHCP ones. -#dhcp-range=1234::2, 1234::500, slaac - -# Do Router Advertisements and stateless DHCP for this subnet. Clients will -# not get addresses from DHCP, but they will get other configuration information. -# They will use SLAAC for addresses. -#dhcp-range=1234::, ra-stateless - -# Do stateless DHCP, SLAAC, and generate DNS names for SLAAC addresses -# from DHCPv4 leases. -#dhcp-range=1234::, ra-stateless, ra-names - -# Do router advertisements for all subnets where we're doing DHCPv6 -# Unless overridden by ra-stateless, ra-names, et al, the router -# advertisements will have the M and O bits set, so that the clients -# get addresses and configuration from DHCPv6, and the A bit reset, so the -# clients don't use SLAAC addresses. -#enable-ra - -# Supply parameters for specified hosts using DHCP. There are lots -# of valid alternatives, so we will give examples of each. Note that -# IP addresses DO NOT have to be in the range given above, they just -# need to be on the same network. The order of the parameters in these -# do not matter, it's permissible to give name, address and MAC in any -# order. - -# Always allocate the host with Ethernet address 11:22:33:44:55:66 -# The IP address 192.168.0.60 -#dhcp-host=11:22:33:44:55:66,192.168.0.60 - -# Always set the name of the host with hardware address -# 11:22:33:44:55:66 to be "fred" -#dhcp-host=11:22:33:44:55:66,fred - -# Always give the host with Ethernet address 11:22:33:44:55:66 -# the name fred and IP address 192.168.0.60 and lease time 45 minutes -#dhcp-host=11:22:33:44:55:66,fred,192.168.0.60,45m - -# Give a host with Ethernet address 11:22:33:44:55:66 or -# 12:34:56:78:90:12 the IP address 192.168.0.60. Dnsmasq will assume -# that these two Ethernet interfaces will never be in use at the same -# time, and give the IP address to the second, even if it is already -# in use by the first. Useful for laptops with wired and wireless -# addresses. -#dhcp-host=11:22:33:44:55:66,12:34:56:78:90:12,192.168.0.60 - -# Give the machine which says its name is "bert" IP address -# 192.168.0.70 and an infinite lease -#dhcp-host=bert,192.168.0.70,infinite - -# Always give the host with client identifier 01:02:02:04 -# the IP address 192.168.0.60 -#dhcp-host=id:01:02:02:04,192.168.0.60 - -# Always give the host with client identifier "marjorie" -# the IP address 192.168.0.60 -#dhcp-host=id:marjorie,192.168.0.60 - -# Enable the address given for "judge" in /etc/hosts -# to be given to a machine presenting the name "judge" when -# it asks for a DHCP lease. -#dhcp-host=judge - -# Never offer DHCP service to a machine whose Ethernet -# address is 11:22:33:44:55:66 -#dhcp-host=11:22:33:44:55:66,ignore - -# Ignore any client-id presented by the machine with Ethernet -# address 11:22:33:44:55:66. This is useful to prevent a machine -# being treated differently when running under different OS's or -# between PXE boot and OS boot. -#dhcp-host=11:22:33:44:55:66,id:* - -# Send extra options which are tagged as "red" to -# the machine with Ethernet address 11:22:33:44:55:66 -#dhcp-host=11:22:33:44:55:66,set:red - -# Send extra options which are tagged as "red" to -# any machine with Ethernet address starting 11:22:33: -#dhcp-host=11:22:33:*:*:*,set:red - -# Give a fixed IPv6 address and name to client with -# DUID 00:01:00:01:16:d2:83:fc:92:d4:19:e2:d8:b2 -# Note the MAC addresses CANNOT be used to identify DHCPv6 clients. -# Note also the they [] around the IPv6 address are obligatory. -#dhcp-host=id:00:01:00:01:16:d2:83:fc:92:d4:19:e2:d8:b2, fred, [1234::5] - -# Ignore any clients which are not specified in dhcp-host lines -# or /etc/ethers. Equivalent to ISC "deny unknown-clients". -# This relies on the special "known" tag which is set when -# a host is matched. -#dhcp-ignore=tag:!known - -# Send extra options which are tagged as "red" to any machine whose -# DHCP vendorclass string includes the substring "Linux" -#dhcp-vendorclass=set:red,Linux - -# Send extra options which are tagged as "red" to any machine one -# of whose DHCP userclass strings includes the substring "accounts" -#dhcp-userclass=set:red,accounts - -# Send extra options which are tagged as "red" to any machine whose -# MAC address matches the pattern. -#dhcp-mac=set:red,00:60:8C:*:*:* - -# If this line is uncommented, dnsmasq will read /etc/ethers and act -# on the ethernet-address/IP pairs found there just as if they had -# been given as --dhcp-host options. Useful if you keep -# MAC-address/host mappings there for other purposes. -#read-ethers - -# Send options to hosts which ask for a DHCP lease. -# See RFC 2132 for details of available options. -# Common options can be given to dnsmasq by name: -# run "dnsmasq --help dhcp" to get a list. -# Note that all the common settings, such as netmask and -# broadcast address, DNS server and default route, are given -# sane defaults by dnsmasq. You very likely will not need -# any dhcp-options. If you use Windows clients and Samba, there -# are some options which are recommended, they are detailed at the -# end of this section. - -# Override the default route supplied by dnsmasq, which assumes the -# router is the same machine as the one running dnsmasq. -#dhcp-option=3,1.2.3.4 - -# Do the same thing, but using the option name -#dhcp-option=option:router,1.2.3.4 - -# Override the default route supplied by dnsmasq and send no default -# route at all. Note that this only works for the options sent by -# default (1, 3, 6, 12, 28) the same line will send a zero-length option -# for all other option numbers. -#dhcp-option=3 - -# Set the NTP time server addresses to 192.168.0.4 and 10.10.0.5 -#dhcp-option=option:ntp-server,192.168.0.4,10.10.0.5 - -# Send DHCPv6 option. Note [] around IPv6 addresses. -#dhcp-option=option6:dns-server,[1234::77],[1234::88] - -# Send DHCPv6 option for namservers as the machine running -# dnsmasq and another. -#dhcp-option=option6:dns-server,[::],[1234::88] - -# Ask client to poll for option changes every six hours. (RFC4242) -#dhcp-option=option6:information-refresh-time,6h - -# Set the NTP time server address to be the same machine as -# is running dnsmasq -#dhcp-option=42,0.0.0.0 - -# Set the NIS domain name to "welly" -#dhcp-option=40,welly - -# Set the default time-to-live to 50 -#dhcp-option=23,50 - -# Set the "all subnets are local" flag -#dhcp-option=27,1 - -# Send the etherboot magic flag and then etherboot options (a string). -#dhcp-option=128,e4:45:74:68:00:00 -#dhcp-option=129,NIC=eepro100 - -# Specify an option which will only be sent to the "red" network -# (see dhcp-range for the declaration of the "red" network) -# Note that the tag: part must precede the option: part. -#dhcp-option = tag:red, option:ntp-server, 192.168.1.1 - -# The following DHCP options set up dnsmasq in the same way as is specified -# for the ISC dhcpcd in -# http://www.samba.org/samba/ftp/docs/textdocs/DHCP-Server-Configuration.txt -# adapted for a typical dnsmasq installation where the host running -# dnsmasq is also the host running samba. -# you may want to uncomment some or all of them if you use -# Windows clients and Samba. -#dhcp-option=19,0 # option ip-forwarding off -#dhcp-option=44,0.0.0.0 # set netbios-over-TCP/IP nameserver(s) aka WINS server(s) -#dhcp-option=45,0.0.0.0 # netbios datagram distribution server -#dhcp-option=46,8 # netbios node type - -# Send an empty WPAD option. This may be REQUIRED to get windows 7 to behave. -#dhcp-option=252,"\n" - -# Send RFC-3397 DNS domain search DHCP option. WARNING: Your DHCP client -# probably doesn't support this...... -#dhcp-option=option:domain-search,eng.apple.com,marketing.apple.com - -# Send RFC-3442 classless static routes (note the netmask encoding) -#dhcp-option=121,192.168.1.0/24,1.2.3.4,10.0.0.0/8,5.6.7.8 - -# Send vendor-class specific options encapsulated in DHCP option 43. -# The meaning of the options is defined by the vendor-class so -# options are sent only when the client supplied vendor class -# matches the class given here. (A substring match is OK, so "MSFT" -# matches "MSFT" and "MSFT 5.0"). This example sets the -# mtftp address to 0.0.0.0 for PXEClients. -#dhcp-option=vendor:PXEClient,1,0.0.0.0 - -# Send microsoft-specific option to tell windows to release the DHCP lease -# when it shuts down. Note the "i" flag, to tell dnsmasq to send the -# value as a four-byte integer - that's what microsoft wants. See -# http://technet2.microsoft.com/WindowsServer/en/library/a70f1bb7-d2d4-49f0-96d6-4b7414ecfaae1033.mspx?mfr=true -#dhcp-option=vendor:MSFT,2,1i - -# Send the Encapsulated-vendor-class ID needed by some configurations of -# Etherboot to allow is to recognize the DHCP server. -#dhcp-option=vendor:Etherboot,60,"Etherboot" - -# Send options to PXELinux. Note that we need to send the options even -# though they don't appear in the parameter request list, so we need -# to use dhcp-option-force here. -# See http://syslinux.zytor.com/pxe.php#special for details. -# Magic number - needed before anything else is recognized -#dhcp-option-force=208,f1:00:74:7e -# Configuration file name -#dhcp-option-force=209,configs/common -# Path prefix -#dhcp-option-force=210,/tftpboot/pxelinux/files/ -# Reboot time. (Note 'i' to send 32-bit value) -#dhcp-option-force=211,30i - -# Set the boot filename for netboot/PXE. You will only need -# this is you want to boot machines over the network and you will need -# a TFTP server; either dnsmasq's built in TFTP server or an -# external one. (See below for how to enable the TFTP server.) -#dhcp-boot=pxelinux.0 - -# The same as above, but use custom tftp-server instead machine running dnsmasq -#dhcp-boot=pxelinux,server.name,192.168.1.100 - -# Boot for Etherboot gPXE. The idea is to send two different -# filenames, the first loads gPXE, and the second tells gPXE what to -# load. The dhcp-match sets the gpxe tag for requests from gPXE. -#dhcp-match=set:gpxe,175 # gPXE sends a 175 option. -#dhcp-boot=tag:!gpxe,undionly.kpxe -#dhcp-boot=mybootimage - -# Encapsulated options for Etherboot gPXE. All the options are -# encapsulated within option 175 -#dhcp-option=encap:175, 1, 5b # priority code -#dhcp-option=encap:175, 176, 1b # no-proxydhcp -#dhcp-option=encap:175, 177, string # bus-id -#dhcp-option=encap:175, 189, 1b # BIOS drive code -#dhcp-option=encap:175, 190, user # iSCSI username -#dhcp-option=encap:175, 191, pass # iSCSI password - -# Test for the architecture of a netboot client. PXE clients are -# supposed to send their architecture as option 93. (See RFC 4578) -#dhcp-match=peecees, option:client-arch, 0 #x86-32 -#dhcp-match=itanics, option:client-arch, 2 #IA64 -#dhcp-match=hammers, option:client-arch, 6 #x86-64 -#dhcp-match=mactels, option:client-arch, 7 #EFI x86-64 - -# Do real PXE, rather than just booting a single file, this is an -# alternative to dhcp-boot. -#pxe-prompt="What system shall I netboot?" -# or with timeout before first available action is taken: -#pxe-prompt="Press F8 for menu.", 60 - -# Available boot services. for PXE. -#pxe-service=x86PC, "Boot from local disk" - -# Loads /pxelinux.0 from dnsmasq TFTP server. -#pxe-service=x86PC, "Install Linux", pxelinux - -# Loads /pxelinux.0 from TFTP server at 1.2.3.4. -# Beware this fails on old PXE ROMS. -#pxe-service=x86PC, "Install Linux", pxelinux, 1.2.3.4 - -# Use bootserver on network, found my multicast or broadcast. -#pxe-service=x86PC, "Install windows from RIS server", 1 - -# Use bootserver at a known IP address. -#pxe-service=x86PC, "Install windows from RIS server", 1, 1.2.3.4 - -# If you have multicast-FTP available, -# information for that can be passed in a similar way using options 1 -# to 5. See page 19 of -# http://download.intel.com/design/archives/wfm/downloads/pxespec.pdf - - -# Enable dnsmasq's built-in TFTP server -#enable-tftp - -# Set the root directory for files available via FTP. -#tftp-root=/var/ftpd - -# Make the TFTP server more secure: with this set, only files owned by -# the user dnsmasq is running as will be send over the net. -#tftp-secure - -# This option stops dnsmasq from negotiating a larger blocksize for TFTP -# transfers. It will slow things down, but may rescue some broken TFTP -# clients. -#tftp-no-blocksize - -# Set the boot file name only when the "red" tag is set. -#dhcp-boot=tag:red,pxelinux.red-net - -# An example of dhcp-boot with an external TFTP server: the name and IP -# address of the server are given after the filename. -# Can fail with old PXE ROMS. Overridden by --pxe-service. -#dhcp-boot=/var/ftpd/pxelinux.0,boothost,192.168.0.3 - -# If there are multiple external tftp servers having a same name -# (using /etc/hosts) then that name can be specified as the -# tftp_servername (the third option to dhcp-boot) and in that -# case dnsmasq resolves this name and returns the resultant IP -# addresses in round robin fashion. This facility can be used to -# load balance the tftp load among a set of servers. -#dhcp-boot=/var/ftpd/pxelinux.0,boothost,tftp_server_name - -# Set the limit on DHCP leases, the default is 150 -#dhcp-lease-max=150 - -# The DHCP server needs somewhere on disk to keep its lease database. -# This defaults to a sane location, but if you want to change it, use -# the line below. -#dhcp-leasefile=/var/lib/misc/dnsmasq.leases - -# Set the DHCP server to authoritative mode. In this mode it will barge in -# and take over the lease for any client which broadcasts on the network, -# whether it has a record of the lease or not. This avoids long timeouts -# when a machine wakes up on a new network. DO NOT enable this if there's -# the slightest chance that you might end up accidentally configuring a DHCP -# server for your campus/company accidentally. The ISC server uses -# the same option, and this URL provides more information: -# http://www.isc.org/files/auth.html -#dhcp-authoritative - -# Run an executable when a DHCP lease is created or destroyed. -# The arguments sent to the script are "add" or "del", -# then the MAC address, the IP address and finally the hostname -# if there is one. -#dhcp-script=/bin/echo - -# Set the cachesize here. -#cache-size=150 - -# If you want to disable negative caching, uncomment this. -#no-negcache - -# Normally responses which come from /etc/hosts and the DHCP lease -# file have Time-To-Live set as zero, which conventionally means -# do not cache further. If you are happy to trade lower load on the -# server for potentially stale date, you can set a time-to-live (in -# seconds) here. -#local-ttl= - -# If you want dnsmasq to detect attempts by Verisign to send queries -# to unregistered .com and .net hosts to its sitefinder service and -# have dnsmasq instead return the correct NXDOMAIN response, uncomment -# this line. You can add similar lines to do the same for other -# registries which have implemented wildcard A records. -#bogus-nxdomain=64.94.110.11 - -# If you want to fix up DNS results from upstream servers, use the -# alias option. This only works for IPv4. -# This alias makes a result of 1.2.3.4 appear as 5.6.7.8 -#alias=1.2.3.4,5.6.7.8 -# and this maps 1.2.3.x to 5.6.7.x -#alias=1.2.3.0,5.6.7.0,255.255.255.0 -# and this maps 192.168.0.10->192.168.0.40 to 10.0.0.10->10.0.0.40 -#alias=192.168.0.10-192.168.0.40,10.0.0.0,255.255.255.0 - -# Change these lines if you want dnsmasq to serve MX records. - -# Return an MX record named "maildomain.com" with target -# servermachine.com and preference 50 -#mx-host=maildomain.com,servermachine.com,50 - -# Set the default target for MX records created using the localmx option. -#mx-target=servermachine.com - -# Return an MX record pointing to the mx-target for all local -# machines. -#localmx - -# Return an MX record pointing to itself for all local machines. -#selfmx - -# Change the following lines if you want dnsmasq to serve SRV -# records. These are useful if you want to serve ldap requests for -# Active Directory and other windows-originated DNS requests. -# See RFC 2782. -# You may add multiple srv-host lines. -# The fields are ,,,, -# If the domain part if missing from the name (so that is just has the -# service and protocol sections) then the domain given by the domain= -# config option is used. (Note that expand-hosts does not need to be -# set for this to work.) - -# A SRV record sending LDAP for the example.com domain to -# ldapserver.example.com port 389 -#srv-host=_ldap._tcp.example.com,ldapserver.example.com,389 - -# A SRV record sending LDAP for the example.com domain to -# ldapserver.example.com port 389 (using domain=) -#domain=example.com -#srv-host=_ldap._tcp,ldapserver.example.com,389 - -# Two SRV records for LDAP, each with different priorities -#srv-host=_ldap._tcp.example.com,ldapserver.example.com,389,1 -#srv-host=_ldap._tcp.example.com,ldapserver.example.com,389,2 - -# A SRV record indicating that there is no LDAP server for the domain -# example.com -#srv-host=_ldap._tcp.example.com - -# The following line shows how to make dnsmasq serve an arbitrary PTR -# record. This is useful for DNS-SD. (Note that the -# domain-name expansion done for SRV records _does_not -# occur for PTR records.) -#ptr-record=_http._tcp.dns-sd-services,"New Employee Page._http._tcp.dns-sd-services" - -# Change the following lines to enable dnsmasq to serve TXT records. -# These are used for things like SPF and zeroconf. (Note that the -# domain-name expansion done for SRV records _does_not -# occur for TXT records.) - -#Example SPF. -#txt-record=example.com,"v=spf1 a -all" - -#Example zeroconf -#txt-record=_http._tcp.example.com,name=value,paper=A4 - -# Provide an alias for a "local" DNS name. Note that this _only_ works -# for targets which are names from DHCP or /etc/hosts. Give host -# "bert" another name, bertrand -#cname=bertand,bert - -# For debugging purposes, log each DNS query as it passes through -# dnsmasq. -#log-queries - -# Log lots of extra information about DHCP transactions. -#log-dhcp - -# Include another lot of configuration options. -#conf-file=/etc/dnsmasq.more.conf -#conf-dir=/etc/dnsmasq.d - -# Include all the files in a directory except those ending in .bak -#conf-dir=/etc/dnsmasq.d,.bak - -# Include all files in a directory which end in .conf -#conf-dir=/etc/dnsmasq.d/*.conf diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index d9e22e08..7d899940 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1137,83 +1137,31 @@ installDefaultBlocklists() { echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}" } -# Check if /etc/dnsmasq.conf is from pi-hole. If so replace with an original and install new in .d directory -version_check_dnsmasq() { +remove_old_dnsmasq_ftl_configs() { # Local, named variables local dnsmasq_conf="/etc/dnsmasq.conf" - local dnsmasq_conf_orig="/etc/dnsmasq.conf.orig" - local dnsmasq_pihole_id_string="addn-hosts=/etc/pihole/gravity.list" - local dnsmasq_pihole_id_string2="# Dnsmasq config for Pi-hole's FTLDNS" - local dnsmasq_original_config="${PI_HOLE_LOCAL_REPO}/advanced/dnsmasq.conf.original" - local dnsmasq_pihole_01_source="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf" - local dnsmasq_pihole_01_target="/etc/dnsmasq.d/01-pihole.conf" - local dnsmasq_rfc6761_06_source="${PI_HOLE_LOCAL_REPO}/advanced/06-rfc6761.conf" - local dnsmasq_rfc6761_06_target="/etc/dnsmasq.d/06-rfc6761.conf" + local pihole_01="/etc/dnsmasq.d/01-pihole.conf" + local rfc6761_06="/etc/dnsmasq.d/06-rfc6761.conf" + local pihole_dhcp_02="/etc/dnsmasq.d/02-pihole-dhcp.conf" + + # pihole-FTL does some fancy stuff with config these days, and so we can remove some old config files + if [[ -f "${pihole_01}" ]]; then + rm "${pihole_01}" + fi + + if [[ -f "${rfc6761_06}" ]]; then + rm "${rfc6761_06}" + fi + + if [[ -f "${pihole_dhcp_02}" ]]; then + rm "${pihole_dhcp_02}" + fi # If the dnsmasq config file exists if [[ -f "${dnsmasq_conf}" ]]; then - printf " %b Existing dnsmasq.conf found..." "${INFO}" - # If a specific string is found within this file, we presume it's from older versions on Pi-hole, - if grep -q "${dnsmasq_pihole_id_string}" "${dnsmasq_conf}" || - grep -q "${dnsmasq_pihole_id_string2}" "${dnsmasq_conf}"; then - printf " it is from a previous Pi-hole install.\\n" - printf " %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}" - # so backup the original file, - mv -f "${dnsmasq_conf}" "${dnsmasq_conf_orig}" - printf "%b %b Backing up dnsmasq.conf to dnsmasq.conf.orig...\\n" "${OVER}" "${TICK}" - printf " %b Restoring default dnsmasq.conf..." "${INFO}" - # and replace it with the default - install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}" - printf "%b %b Restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" - else - # Otherwise, don't to anything - printf " it is not a Pi-hole file, leaving alone!\\n" - fi - else - # If a file cannot be found, - printf " %b No dnsmasq.conf found... restoring default dnsmasq.conf..." "${INFO}" - # restore the default one - install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}" - printf "%b %b No dnsmasq.conf found... restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" + # Back it up - we will need to add a symlink to /etc/pihole/dnsmasq.conf later + mv "${dnsmasq_conf}" "${dnsmasq_conf}.old" fi - - printf " %b Installing %s..." "${INFO}" "${dnsmasq_pihole_01_target}" - # Check to see if dnsmasq directory exists (it may not due to being a fresh install and dnsmasq no longer being a dependency) - if [[ ! -d "/etc/dnsmasq.d" ]];then - install -d -m 755 "/etc/dnsmasq.d" - fi - # Copy the new Pi-hole DNS config file into the dnsmasq.d directory - install -D -m 644 -T "${dnsmasq_pihole_01_source}" "${dnsmasq_pihole_01_target}" - printf "%b %b Installed %s\n" "${OVER}" "${TICK}" "${dnsmasq_pihole_01_target}" - # Add settings with the GLOBAL DNS variables that we populated earlier - # First, set the interface to listen on - addOrEditKeyValPair "${dnsmasq_pihole_01_target}" "interface" "$PIHOLE_INTERFACE" - if [[ "${PIHOLE_DNS_1}" != "" ]]; then - # then add in the primary DNS server. - addOrEditKeyValPair "${dnsmasq_pihole_01_target}" "server" "$PIHOLE_DNS_1" - fi - # Ditto if DNS2 is not empty - if [[ "${PIHOLE_DNS_2}" != "" ]]; then - addKey "${dnsmasq_pihole_01_target}" "server=$PIHOLE_DNS_2" - fi - - # Set the cache size - addOrEditKeyValPair "${dnsmasq_pihole_01_target}" "cache-size" "$CACHE_SIZE" - - sed -i 's/^#conf-dir=\/etc\/dnsmasq.d$/conf-dir=\/etc\/dnsmasq.d/' "${dnsmasq_conf}" - - # If the user does not want to enable logging, - if [[ "${QUERY_LOGGING}" == false ]] ; then - # remove itfrom the DNS config file - removeKey "${dnsmasq_pihole_01_target}" "log-queries" - else - # Otherwise, enable it by adding the directive to the DNS config file - addKey "${dnsmasq_pihole_01_target}" "log-queries" - fi - - printf " %b Installing %s..." "${INFO}" "${dnsmasq_rfc6761_06_source}" - install -D -m 644 -T "${dnsmasq_rfc6761_06_source}" "${dnsmasq_rfc6761_06_target}" - printf "%b %b Installed %s\n" "${OVER}" "${TICK}" "${dnsmasq_rfc6761_06_target}" } # Clean an existing installation to prepare for upgrade/reinstall @@ -1272,7 +1220,7 @@ installScripts() { installConfigs() { printf "\\n %b Installing configs from %s...\\n" "${INFO}" "${PI_HOLE_LOCAL_REPO}" # Make sure Pi-hole's config files are in place - version_check_dnsmasq + remove_old_dnsmasq_ftl_configs # Install list of DNS servers # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6 @@ -2077,28 +2025,6 @@ FTLinstall() { fi } -disable_dnsmasq() { - # dnsmasq can now be stopped and disabled if it exists - if is_command dnsmasq; then - if check_service_active "dnsmasq";then - printf " %b FTL can now resolve DNS Queries without dnsmasq running separately\\n" "${INFO}" - stop_service dnsmasq - disable_service dnsmasq - fi - fi - - # Backup existing /etc/dnsmasq.conf if present and ensure that - # /etc/dnsmasq.conf contains only "conf-dir=/etc/dnsmasq.d" - local conffile="/etc/dnsmasq.conf" - if [[ -f "${conffile}" ]]; then - printf " %b Backing up %s to %s.old\\n" "${INFO}" "${conffile}" "${conffile}" - mv "${conffile}" "${conffile}.old" - fi - # Create /etc/dnsmasq.conf - echo "conf-dir=/etc/dnsmasq.d" > "${conffile}" - chmod 644 "${conffile}" -} - get_binary_name() { # This gives the machine architecture which may be different from the OS architecture... local machine From 0e558e4c367132f2c525a27ec7bf19c2ffe10e6c Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 13 Jan 2023 21:30:01 +0000 Subject: [PATCH 068/462] Remove some code that installs the blockpage/sets up web directories Remove some code that disables dnsmasq if it is running Create dnsmasq.d directory if it does not exist --- automated install/basic-install.sh | 28 +++++++++------------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 7d899940..9d9eb8be 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1162,6 +1162,11 @@ remove_old_dnsmasq_ftl_configs() { # Back it up - we will need to add a symlink to /etc/pihole/dnsmasq.conf later mv "${dnsmasq_conf}" "${dnsmasq_conf}.old" fi + + # Create /etc/dnsmasq.d if it doesn't exist + if [[ ! -d "/etc/dnsmasq.d" ]]; then + mkdir "/etc/dnsmasq.d" + fi } # Clean an existing installation to prepare for upgrade/reinstall @@ -1380,7 +1385,7 @@ check_service_active() { fi } -# Systemd-resolved's DNSStubListener and dnsmasq can't share port 53. +# Systemd-resolved's DNSStubListener and ftl can't share port 53. disable_resolved_stublistener() { printf " %b Testing if systemd-resolved is enabled\\n" "${INFO}" # Check if Systemd-resolved's DNSStubListener is enabled and active on port 53 @@ -1389,7 +1394,7 @@ disable_resolved_stublistener() { printf " %b %b Testing if systemd-resolved DNSStub-Listener is active" "${OVER}" "${INFO}" if ( grep -E '#?DNSStubListener=yes' /etc/systemd/resolved.conf &> /dev/null ); then # Disable the DNSStubListener to unbind it from port 53 - # Note that this breaks dns functionality on host until dnsmasq/ftl are up and running + # Note that this breaks dns functionality on host until ftl are up and running printf "%b %b Disabling systemd-resolved DNSStubListener" "${OVER}" "${TICK}" # Make a backup of the original /etc/systemd/resolved.conf # (This will need to be restored on uninstallation) @@ -1695,21 +1700,13 @@ installPihole() { printf " %b Failure in dependent config copy function.\\n" "${CROSS}" exit 1 fi - # If the user wants to install the dashboard, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # do so - installPiholeWeb - fi + # Install the cron file installCron # Install the logrotate file installLogrotate || true - # Check if dnsmasq is present. If so, disable it and back up any possible - # config file - disable_dnsmasq - # install a man page entry for pihole install_manpage @@ -2134,13 +2131,6 @@ FTLcheckUpdate() { local remoteSha1 local localSha1 - # if dnsmasq exists and is running at this point, force reinstall of FTL Binary - if is_command dnsmasq; then - if check_service_active "dnsmasq";then - return 0 - fi - fi - if [[ ! "${ftlBranch}" == "master" ]]; then #Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! local path @@ -2397,7 +2387,7 @@ main() { # Check for and disable systemd-resolved-DNSStubListener before reloading resolved # DNSStubListener needs to remain in place for installer to download needed files, # so this change needs to be made after installation is complete, - # but before starting or resarting the dnsmasq or ftl services + # but before starting or resarting the ftl service disable_resolved_stublistener printf " %b Restarting services...\\n" "${INFO}" From e3ffec57622c45ed01e1a9bf8e1ee31159a35871 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 13 Jan 2023 22:23:33 +0000 Subject: [PATCH 069/462] Change when old configs are removed Signed-off-by: Adam Warner --- automated install/basic-install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9d9eb8be..764503f2 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1224,8 +1224,7 @@ installScripts() { # Install the configs from PI_HOLE_LOCAL_REPO to their various locations installConfigs() { printf "\\n %b Installing configs from %s...\\n" "${INFO}" "${PI_HOLE_LOCAL_REPO}" - # Make sure Pi-hole's config files are in place - remove_old_dnsmasq_ftl_configs + # Install list of DNS servers # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6 @@ -1695,6 +1694,8 @@ installPihole() { exit 1 fi + remove_old_dnsmasq_ftl_configs + # Install config files if ! installConfigs; then printf " %b Failure in dependent config copy function.\\n" "${CROSS}" From 632aead691f747036c8d78a1096d705cf26820e5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 14 Jan 2023 23:11:42 +0000 Subject: [PATCH 070/462] add wrappers to utils.sh for setting FTL config & add tests Signed-off-by: Adam Warner --- advanced/Scripts/utils.sh | 23 ++++++++++++++++ test/test_any_utils.py | 55 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index f655e56c..c97ad0cf 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -145,3 +145,26 @@ getFTLPID() { FTL_PID=${FTL_PID:=-1} echo "${FTL_PID}" } + +####################### +# returns value from FTLs config file using pihole-FTL --config +# +# Takes one argument: key +# Example getFTLConfigValue dns.piholePTR +####################### +getFTLConfigValue(){ + pihole-FTL --config "${1}" +} + +####################### +# sets value in FTLs config file using pihole-FTL --config +# +# Takes two arguments: key and value +# Example setFTLConfigValue dns.piholePTR PI.HOLE +# +# Note, for complex values such as dnsmasq.upstreams, you should wrap the value in single quotes: +# setFTLConfigValue dnsmasq.upstreams '[ "8.8.8.8" , "8.8.4.4" ]' +####################### +setFTLConfigValue(){ + pihole-FTL --config "${1}" "${2}" +} diff --git a/test/test_any_utils.py b/test/test_any_utils.py index b3fabe6c..741b1127 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -170,3 +170,58 @@ def test_getFTLPIDFile_and_getFTLPID_custom(host): ) expected_stdout = "1234\n" assert expected_stdout == output.stdout + + +def test_setFTLConfigValue_getFTLConfigValue(host): + """ + Confirms setFTLConfigValue works + Requires FTL to be installed, so we do that first (taken from test_FTL_binary_installed_and_responsive_no_errors) + """ + host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + echo "new/http" > /etc/pihole/ftlbranch + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + + output = host.run( + """ + source /opt/pihole/utils.sh + setFTLConfigValue "dnsmasq.upstreams" '["1.1.1.1"]' + """ + ) + + assert '[ "1.1.1.1" ]' in output.stdout + + +def test_getFTLConfigValue_getFTLConfigValue(host): + """ + Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) + Requires FTL to be installed, so we do that first (taken from test_FTL_binary_installed_and_responsive_no_errors) + """ + host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + echo "new/http" > /etc/pihole/ftlbranch + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + + output = host.run( + """ + source /opt/pihole/utils.sh + setFTLConfigValue "dnsmasq.upstreams" '["9.9.9.9"]' > /dev/null + getFTLConfigValue "dnsmasq.upstreams" + """ + ) + + assert '[ "9.9.9.9" ]' in output.stdout From 019be067d9ddaf5cf1e1304ddb53c69ce2237a5e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 14 Jan 2023 23:15:40 +0000 Subject: [PATCH 071/462] Convert SetWebPassword to use new FTL config Signed-off-by: Adam Warner --- advanced/Scripts/webpage.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 7680590f..1282b1ee 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -77,7 +77,7 @@ deleteFTLsetting() { } changeFTLsetting() { - addOrEditKeyValPair "${FTLconf}" "${1}" "${2}" + pihole-FTL --config "${1}" "${2}" } add_dnsmasq_setting() { @@ -165,7 +165,7 @@ SetWebPassword() { echo "" if [ "${PASSWORD}" == "" ]; then - addOrEditKeyValPair "${setupVars}" "WEBPASSWORD" "" + setFTLConfigValue "api.pwhash" "" echo -e " ${TICK} Password Removed" exit 0 fi @@ -178,7 +178,7 @@ SetWebPassword() { # We do not wrap this in brackets, otherwise BASH will expand any appropriate syntax hash=$(HashPassword "$PASSWORD") # Save hash to file - addOrEditKeyValPair "${setupVars}" "WEBPASSWORD" "${hash}" + setFTLConfigValue "api.pwhash" "${hash}" echo -e " ${TICK} New password set" else echo -e " ${CROSS} Passwords don't match. Your password has not been changed" From b8acccde90502731e1a1226a45ff8e4c903d0421 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 11:33:53 +0000 Subject: [PATCH 072/462] Update comments on the move of /etc/dnsmasq.old -> /etc/dnsmasq.old to better reflect why we are doing it Signed-off-by: Adam Warner --- automated install/basic-install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 764503f2..87ff6d83 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1159,7 +1159,8 @@ remove_old_dnsmasq_ftl_configs() { # If the dnsmasq config file exists if [[ -f "${dnsmasq_conf}" ]]; then - # Back it up - we will need to add a symlink to /etc/pihole/dnsmasq.conf later + # There should not be anything custom in here for Pi-hole users + # It is no longer needed, but we'll back it up instead of deleting it just in case mv "${dnsmasq_conf}" "${dnsmasq_conf}.old" fi From 8f7c82840728ac742e21eb0fd260243c6f60416f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 13:47:38 +0000 Subject: [PATCH 073/462] No need for test_installPiholeWeb_fresh_install_no_errors Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index c1288287..e15839f2 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -118,21 +118,6 @@ def test_selinux_not_detected(host): assert check_selinux.rc == 0 -def test_installPiholeWeb_fresh_install_no_errors(host): - """ - confirms all web page assets from Core repo are installed on a fresh build - """ - installWeb = host.run( - """ - umask 0027 - source /opt/pihole/basic-install.sh - installPiholeWeb - """ - ) - expected_stdout = tick_box + " Installing sudoer file" - assert expected_stdout in installWeb.stdout - - def get_directories_recursive(host, directory): if directory is None: return directory From 5e431210fd5a312939a4e204f0d4e32df199b744 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 13:49:25 +0000 Subject: [PATCH 074/462] [REVISIT] Remove test_setupVars_saved_to_file for now - it may no longer be needed as we move towards getting rid of the file. Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 34 ------------------------------ 1 file changed, 34 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index e15839f2..059e3a8b 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -68,40 +68,6 @@ def test_setupVars_are_sourced_to_global_scope(host): assert "{}={}".format(k, v) in output -def test_setupVars_saved_to_file(host): - """ - confirm saved settings are written to a file for future updates to re-use - """ - # dedent works better with this and padding matching script below - set_setup_vars = "\n" - for k, v in SETUPVARS.items(): - set_setup_vars += " {}={}\n".format(k, v) - host.run(set_setup_vars) - - script = dedent( - """\ - set -e - echo start - TERM=xterm - source /opt/pihole/basic-install.sh - source /opt/pihole/utils.sh - {} - mkdir -p /etc/dnsmasq.d - version_check_dnsmasq - echo "" > /etc/pihole/pihole-FTL.conf - finalExports - cat /etc/pihole/setupVars.conf - """.format( - set_setup_vars - ) - ) - - output = run_script(host, script).stdout - - for k, v in SETUPVARS.items(): - assert "{}={}".format(k, v) in output - - def test_selinux_not_detected(host): """ confirms installer continues when SELinux configuration file does not exist From 414df5b3724d3a5f5f6cb54d140bb737b27034a5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 13:54:40 +0000 Subject: [PATCH 075/462] Remove no longer required checks in test_installPihole_fresh_install_readableFiles Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 43 ------------------------------ 1 file changed, 43 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 059e3a8b..ef99d57b 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -105,9 +105,6 @@ def test_installPihole_fresh_install_readableFiles(host): mock_command_2( "systemctl", { - "enable lighttpd": ("", "0"), - "restart lighttpd": ("", "0"), - "start lighttpd": ("", "0"), "enable pihole-FTL": ("", "0"), "restart pihole-FTL": ("", "0"), "start pihole-FTL": ("", "0"), @@ -123,7 +120,6 @@ def test_installPihole_fresh_install_readableFiles(host): setup_var_file = "cat < /etc/pihole/setupVars.conf\n" for k, v in SETUPVARS.items(): setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "INSTALL_WEB_SERVER=true\n" setup_var_file += "INSTALL_WEB_INTERFACE=true\n" setup_var_file += "EOF\n" host.run(setup_var_file) @@ -194,23 +190,6 @@ def test_installPihole_fresh_install_readableFiles(host): check_setup = test_cmd.format("r", "/etc/pihole/setupVars.conf", piholeuser) actual_rc = host.run(check_setup).rc assert exit_status_success == actual_rc - # check dnsmasq files - # readable /etc/dnsmasq.conf - check_dnsmasqconf = test_cmd.format("r", "/etc/dnsmasq.conf", piholeuser) - actual_rc = host.run(check_dnsmasqconf).rc - assert exit_status_success == actual_rc - # readable /etc/dnsmasq.d/01-pihole.conf - check_dnsmasqconf = test_cmd.format("r", "/etc/dnsmasq.d", piholeuser) - actual_rc = host.run(check_dnsmasqconf).rc - assert exit_status_success == actual_rc - check_dnsmasqconf = test_cmd.format("x", "/etc/dnsmasq.d", piholeuser) - actual_rc = host.run(check_dnsmasqconf).rc - assert exit_status_success == actual_rc - check_dnsmasqconf = test_cmd.format( - "r", "/etc/dnsmasq.d/01-pihole.conf", piholeuser - ) - actual_rc = host.run(check_dnsmasqconf).rc - assert exit_status_success == actual_rc # check readable and executable /etc/init.d/pihole-FTL check_init = test_cmd.format("x", "/etc/init.d/pihole-FTL", piholeuser) actual_rc = host.run(check_init).rc @@ -218,28 +197,6 @@ def test_installPihole_fresh_install_readableFiles(host): check_init = test_cmd.format("r", "/etc/init.d/pihole-FTL", piholeuser) actual_rc = host.run(check_init).rc assert exit_status_success == actual_rc - # check readable /etc/lighttpd/lighttpd.conf - check_lighttpd = test_cmd.format("r", "/etc/lighttpd/lighttpd.conf", piholeuser) - actual_rc = host.run(check_lighttpd).rc - assert exit_status_success == actual_rc - # check readable /etc/lighttpd/conf*/pihole-admin.conf - check_lighttpd = test_cmd.format("r", "/etc/lighttpd/conf.d", piholeuser) - if host.run(check_lighttpd).rc == exit_status_success: - check_lighttpd = test_cmd.format( - "r", "/etc/lighttpd/conf.d/pihole-admin.conf", piholeuser - ) - actual_rc = host.run(check_lighttpd).rc - assert exit_status_success == actual_rc - else: - check_lighttpd = test_cmd.format( - "r", "/etc/lighttpd/conf-available", piholeuser - ) - if host.run(check_lighttpd).rc == exit_status_success: - check_lighttpd = test_cmd.format( - "r", "/etc/lighttpd/conf-available/15-pihole-admin.conf", piholeuser - ) - actual_rc = host.run(check_lighttpd).rc - assert exit_status_success == actual_rc # check readable and executable manpages if maninstalled is True: check_man = test_cmd.format("x", "/usr/local/share/man", piholeuser) From 9bf372ef43f314cfee4770640725376d40cc6681 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 13:58:05 +0000 Subject: [PATCH 076/462] [REVISIT] Remove test_installPihole_fresh_install_readableBlockpage for now. We may be able to recylcle it later, but I have my doubts Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 225 ----------------------------- 1 file changed, 225 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index ef99d57b..66d00814 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -255,231 +255,6 @@ def test_installPihole_fresh_install_readableFiles(host): actual_rc = host.run(check_pihole).rc -@pytest.mark.parametrize("test_webpage", [True]) -def test_installPihole_fresh_install_readableBlockpage(host, test_webpage): - """ - confirms all web page assets from Core repo are readable - by $LIGHTTPD_USER on a fresh build - """ - piholeWebpage = [ - "127.0.0.1", - # "pi.hole" - ] - # dialog returns Cancel for user prompt - mock_command("dialog", {"*": ("", "0")}, host) - - # mock git pull - mock_command_passthrough("git", {"pull": ("", "0")}, host) - # mock systemctl to start lighttpd and FTL - ligthttpdcommand = dedent( - r'''\"\" - echo 'starting lighttpd with {}' - if [ command -v "apt-get" >/dev/null 2>&1 ]; then - LIGHTTPD_USER="www-data" - LIGHTTPD_GROUP="www-data" - else - LIGHTTPD_USER="lighttpd" - LIGHTTPD_GROUP="lighttpd" - fi - mkdir -p "{run}" - chown {usergroup} "{run}" - mkdir -p "{cache}" - chown {usergroup} "/var/cache" - chown {usergroup} "{cache}" - mkdir -p "{compress}" - chown {usergroup} "{compress}" - mkdir -p "{uploads}" - chown {usergroup} "{uploads}" - chmod 0777 /var - chmod 0777 /var/cache - chmod 0777 "{cache}" - find "{run}" -type d -exec chmod 0777 {chmodarg} \;; - find "{run}" -type f -exec chmod 0666 {chmodarg} \;; - find "{compress}" -type d -exec chmod 0777 {chmodarg} \;; - find "{compress}" -type f -exec chmod 0666 {chmodarg} \;; - find "{uploads}" -type d -exec chmod 0777 {chmodarg} \;; - find "{uploads}" -type f -exec chmod 0666 {chmodarg} \;; - /usr/sbin/lighttpd -tt -f '{config}' - /usr/sbin/lighttpd -f '{config}' - echo \"\"'''.format( - "{}", - usergroup="${{LIGHTTPD_USER}}:${{LIGHTTPD_GROUP}}", - chmodarg="{{}}", - config="/etc/lighttpd/lighttpd.conf", - run="/run/lighttpd", - cache="/var/cache/lighttpd", - uploads="/var/cache/lighttpd/uploads", - compress="/var/cache/lighttpd/compress", - ) - ) - FTLcommand = dedent( - '''\"\" - set -x - /etc/init.d/pihole-FTL restart - echo \"\"''' - ) - mock_command_run( - "systemctl", - { - "enable lighttpd": ("", "0"), - "restart lighttpd": (ligthttpdcommand.format("restart"), "0"), - "start lighttpd": (ligthttpdcommand.format("start"), "0"), - "enable pihole-FTL": ("", "0"), - "restart pihole-FTL": (FTLcommand, "0"), - "start pihole-FTL": (FTLcommand, "0"), - "*": ('echo "systemctl call with $@"', "0"), - }, - host, - ) - # create configuration file - setup_var_file = "cat < /etc/pihole/setupVars.conf\n" - for k, v in SETUPVARS.items(): - setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "INSTALL_WEB_SERVER=true\n" - setup_var_file += "INSTALL_WEB_INTERFACE=true\n" - setup_var_file += "EOF\n" - host.run(setup_var_file) - installWeb = host.run( - """ - export TERM=xterm - export DEBIAN_FRONTEND=noninteractive - umask 0027 - runUnattended=true - useUpdateVars=true - source /opt/pihole/basic-install.sh > /dev/null - runUnattended=true - useUpdateVars=true - main - echo "LIGHTTPD_USER=${LIGHTTPD_USER}" - echo "webroot=${webroot}" - echo "INSTALL_WEB_INTERFACE=${INSTALL_WEB_INTERFACE}" - echo "INSTALL_WEB_SERVER=${INSTALL_WEB_SERVER}" - """ - ) - assert 0 == installWeb.rc - piholeuser = "pihole" - webuser = "" - user = re.findall(r"^\s*LIGHTTPD_USER=.*$", installWeb.stdout, re.MULTILINE) - for match in user: - webuser = match.replace("LIGHTTPD_USER=", "").strip() - webroot = "" - user = re.findall(r"^\s*webroot=.*$", installWeb.stdout, re.MULTILINE) - for match in user: - webroot = match.replace("webroot=", "").strip() - if not webroot.strip(): - webroot = "/var/www/html" - installWebInterface = True - interface = re.findall( - r"^\s*INSTALL_WEB_INTERFACE=.*$", installWeb.stdout, re.MULTILINE - ) - for match in interface: - testvalue = match.replace("INSTALL_WEB_INTERFACE=", "").strip().lower() - if not testvalue.strip(): - installWebInterface = testvalue == "true" - installWebServer = True - server = re.findall(r"^\s*INSTALL_WEB_SERVER=.*$", installWeb.stdout, re.MULTILINE) - for match in server: - testvalue = match.replace("INSTALL_WEB_SERVER=", "").strip().lower() - if not testvalue.strip(): - installWebServer = testvalue == "true" - # if webserver install was not requested - # at least pihole must be able to read files - if installWebServer is False: - webuser = piholeuser - exit_status_success = 0 - test_cmd = 'su --shell /bin/bash --command "test -{0} {1}" -p {2}' - # check files that need a running FTL to be created - # readable and writeable pihole-FTL.db - check_FTLconf = test_cmd.format("r", "/etc/pihole/pihole-FTL.db", piholeuser) - actual_rc = host.run(check_FTLconf).rc - assert exit_status_success == actual_rc - check_FTLconf = test_cmd.format("w", "/etc/pihole/pihole-FTL.db", piholeuser) - actual_rc = host.run(check_FTLconf).rc - assert exit_status_success == actual_rc - # check directories above $webroot for read and execute permission - check_var = test_cmd.format("r", "/var", webuser) - actual_rc = host.run(check_var).rc - assert exit_status_success == actual_rc - check_var = test_cmd.format("x", "/var", webuser) - actual_rc = host.run(check_var).rc - assert exit_status_success == actual_rc - check_www = test_cmd.format("r", "/var/www", webuser) - actual_rc = host.run(check_www).rc - assert exit_status_success == actual_rc - check_www = test_cmd.format("x", "/var/www", webuser) - actual_rc = host.run(check_www).rc - assert exit_status_success == actual_rc - check_html = test_cmd.format("r", "/var/www/html", webuser) - actual_rc = host.run(check_html).rc - assert exit_status_success == actual_rc - check_html = test_cmd.format("x", "/var/www/html", webuser) - actual_rc = host.run(check_html).rc - assert exit_status_success == actual_rc - # check directories below $webroot for read and execute permission - check_admin = test_cmd.format("r", webroot + "/admin", webuser) - actual_rc = host.run(check_admin).rc - assert exit_status_success == actual_rc - check_admin = test_cmd.format("x", webroot + "/admin", webuser) - actual_rc = host.run(check_admin).rc - assert exit_status_success == actual_rc - directories = get_directories_recursive(host, webroot + "/admin/") - for directory in directories: - check_pihole = test_cmd.format("r", directory, webuser) - actual_rc = host.run(check_pihole).rc - check_pihole = test_cmd.format("x", directory, webuser) - actual_rc = host.run(check_pihole).rc - findfiles = 'find "{}" -maxdepth 1 -type f -exec echo {{}} \\;;' - filelist = host.run(findfiles.format(directory)) - files = list(filter(bool, filelist.stdout.splitlines())) - for file in files: - check_pihole = test_cmd.format("r", file, webuser) - actual_rc = host.run(check_pihole).rc - # check web interface files - # change nameserver to pi-hole - # setting nameserver in /etc/resolv.conf to pi-hole does - # not work here because of the way docker uses this file - ns = host.run(r"sed -i 's/nameserver.*/nameserver 127.0.0.1/' /etc/resolv.conf") - pihole_is_ns = ns.rc == 0 - - def is_ip(address): - m = re.match(r"(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})", address) - return bool(m) - - if installWebInterface is True: - if test_webpage is True: - # check webpage for unreadable files - noPHPfopen = re.compile( - ( - r"PHP Error(%d+):\s+fopen([^)]+):\s+" - + r"failed to open stream: " - + r"Permission denied in" - ), - re.I, - ) - # using cURL option --dns-servers is not possible - status = ( - 'curl -s --head "{}" | ' - + "head -n 1 | " - + 'grep "HTTP/1.[01] [23].." > /dev/null' - ) - digcommand = r"dig A +short {} @127.0.0.1 | head -n 1" - pagecontent = 'curl --verbose -L "{}"' - for page in piholeWebpage: - testpage = "http://" + page + "/admin/" - resolvesuccess = True - if is_ip(page) is False: - dig = host.run(digcommand.format(page)) - testpage = "http://" + dig.stdout.strip() + "/admin/" - resolvesuccess = dig.rc == 0 - if resolvesuccess or pihole_is_ns: - # check HTTP status of blockpage - actual_rc = host.run(status.format(testpage)) - assert exit_status_success == actual_rc.rc - # check for PHP error - actual_output = host.run(pagecontent.format(testpage)) - assert noPHPfopen.match(actual_output.stdout) is None - - def test_update_package_cache_success_no_errors(host): """ confirms package cache was updated without any errors From bf16fe4a376c30a84f842ec569327ef5dc28b41d Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 15 Jan 2023 15:47:10 +0000 Subject: [PATCH 077/462] Remove the EPEL related tests on centos/fedora - no longer neccasery Signed-off-by: Adam Warner --- test/test_centos_common_support.py | 27 --------------------------- test/test_fedora_support.py | 15 --------------- test/tox.centos_8.ini | 2 +- test/tox.centos_9.ini | 2 +- test/tox.fedora_36.ini | 2 +- test/tox.fedora_37.ini | 2 +- 6 files changed, 4 insertions(+), 46 deletions(-) delete mode 100644 test/test_centos_common_support.py delete mode 100644 test/test_fedora_support.py diff --git a/test/test_centos_common_support.py b/test/test_centos_common_support.py deleted file mode 100644 index 871fee29..00000000 --- a/test/test_centos_common_support.py +++ /dev/null @@ -1,27 +0,0 @@ -import pytest -from .conftest import ( - tick_box, - info_box, - cross_box, - mock_command, -) - - -def test_enable_epel_repository_centos(host): - """ - confirms the EPEL package repository is enabled when installed on CentOS - """ - package_manager_detect = host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - """ - ) - expected_stdout = info_box + ( - " Enabling EPEL package repository " "(https://fedoraproject.org/wiki/EPEL)" - ) - assert expected_stdout in package_manager_detect.stdout - expected_stdout = tick_box + " Installed" - assert expected_stdout in package_manager_detect.stdout - epel_package = host.package("epel-release") - assert epel_package.is_installed diff --git a/test/test_fedora_support.py b/test/test_fedora_support.py deleted file mode 100644 index e7d31a5d..00000000 --- a/test/test_fedora_support.py +++ /dev/null @@ -1,15 +0,0 @@ -def test_epel_and_remi_not_installed_fedora(host): - """ - confirms installer does not attempt to install EPEL/REMI repositories - on Fedora - """ - package_manager_detect = host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - """ - ) - assert package_manager_detect.stdout == "" - - epel_package = host.package("epel-release") - assert not epel_package.is_installed diff --git a/test/tox.centos_8.ini b/test/tox.centos_8.ini index dca77c93..85ae1ffb 100644 --- a/test/tox.centos_8.ini +++ b/test/tox.centos_8.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _centos_8.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.centos_9.ini b/test/tox.centos_9.ini index a69c336a..4c51aefa 100644 --- a/test/tox.centos_9.ini +++ b/test/tox.centos_9.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _centos_9.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_36.ini index 515487ed..1896a45f 100644 --- a/test/tox.fedora_36.ini +++ b/test/tox.fedora_36.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini index 2a8ef398..9c8752cc 100644 --- a/test/tox.fedora_37.ini +++ b/test/tox.fedora_37.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From 36956103004a99f59aa2258ff0880745d535f354 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 24 Jan 2023 19:51:01 +0100 Subject: [PATCH 078/462] Allow running pihole -g without root (run as user pihole) Signed-off-by: DL6ER --- gravity.sh | 2 +- pihole | 20 ++++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index fe2097dd..5b950b28 100755 --- a/gravity.sh +++ b/gravity.sh @@ -912,7 +912,7 @@ gravity_ShowCount # Determine if DNS has been restarted by this instance of gravity if [[ -z "${dnsWasOffline:-}" ]]; then - "${PIHOLE_COMMAND}" restartdns reload + "${PIHOLE_COMMAND}" reloaddns fi gravity_Cleanup diff --git a/pihole b/pihole index 1d9ad82c..6796acc6 100755 --- a/pihole +++ b/pihole @@ -513,13 +513,13 @@ if [[ $# = 0 ]]; then fi # functions that do not require sudo power +need_root=1 case "${1}" in "-h" | "help" | "--help" ) helpFunc;; "-v" | "version" ) versionFunc "$@";; "-c" | "chronometer" ) chronometerFunc "$@";; "-q" | "query" ) queryFunc "$@";; "status" ) statusFunc "$2";; - "tricorder" ) tricorderFunc;; # we need to add all arguments that require sudo power to not trigger the * argument @@ -532,13 +532,14 @@ case "${1}" in "-f" | "flush" ) ;; "-up" | "updatePihole" ) ;; "-r" | "reconfigure" ) ;; - "-g" | "updateGravity" ) ;; "-l" | "logging" ) ;; "uninstall" ) ;; "enable" ) ;; "disable" ) ;; "-d" | "debug" ) ;; "restartdns" ) ;; + "-g" | "updateGravity" ) need_root=0;; + "reloaddns" ) need_root=0;; "-a" | "admin" ) ;; "checkout" ) ;; "updatechecker" ) ;; @@ -547,8 +548,8 @@ case "${1}" in * ) helpFunc;; esac -# Must be root to use this tool -if [[ ! $EUID -eq 0 ]];then +# Must be root to use this tool for most functions +if [[ ! $EUID -eq 0 && need_root -eq 1 ]];then if [[ -x "$(command -v sudo)" ]]; then exec sudo bash "$0" "$@" exit $? @@ -557,6 +558,16 @@ if [[ ! $EUID -eq 0 ]];then exit 1 fi fi +# Can also be user pihole for other functions +if [[ ${USER} != "pihole" && need_root -eq 0 ]];then + if [[ -x "$(command -v sudo)" ]]; then + exec sudo -u pihole bash "$0" "$@" + exit $? + else + echo -e " ${CROSS} sudo is needed to run pihole commands. Please run this script as root or install sudo." + exit 1 + fi +fi # Handle redirecting to specific functions based on arguments case "${1}" in @@ -576,6 +587,7 @@ case "${1}" in "enable" ) piholeEnable 1;; "disable" ) piholeEnable 0 "$2";; "restartdns" ) restartDNS "$2";; + "reloaddns" ) restartDNS "reload";; "-a" | "admin" ) webpageFunc "$@";; "checkout" ) piholeCheckoutFunc "$@";; "updatechecker" ) shift; updateCheckFunc "$@";; From 3cb3adc5ca5796c6b4f79b943eb5563c1c0972a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 3 Feb 2023 19:55:55 +0100 Subject: [PATCH 079/462] Fix setting webpassword via pihole -a -p MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 1282b1ee..53f36a20 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -165,7 +165,7 @@ SetWebPassword() { echo "" if [ "${PASSWORD}" == "" ]; then - setFTLConfigValue "api.pwhash" "" + setFTLConfigValue "webserver.api.pwhash" "" >/dev/null echo -e " ${TICK} Password Removed" exit 0 fi @@ -178,7 +178,7 @@ SetWebPassword() { # We do not wrap this in brackets, otherwise BASH will expand any appropriate syntax hash=$(HashPassword "$PASSWORD") # Save hash to file - setFTLConfigValue "api.pwhash" "${hash}" + setFTLConfigValue "webserver.api.pwhash" "${hash}" >/dev/null echo -e " ${TICK} New password set" else echo -e " ${CROSS} Passwords don't match. Your password has not been changed" From 0568a69d8346d31372a1f35acc9f246369064001 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 13:15:47 +0000 Subject: [PATCH 080/462] Use WEBPORT instead of PRIVACY_LEVEL to set the web port... Signed-off-by: Adam Warner --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 87ff6d83..bce7aa91 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1625,7 +1625,7 @@ finalExports() { addOrEditKeyValPair "${FTL_CONFIG_FILE}" "PRIVACYLEVEL" "${PRIVACY_LEVEL}" # Set the web port - addOrEditKeyValPair "${FTL_CONFIG_FILE}" "WEBPORT" "${PRIVACY_LEVEL}" + addOrEditKeyValPair "${FTL_CONFIG_FILE}" "WEBPORT" "${WEBPORT}" # Bring in the current settings and the functions to manipulate them source "${setupVars}" From dbc6b814b2381e9db3d0361dff7cc7b80ef0236f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 15:15:55 +0000 Subject: [PATCH 081/462] Should not need to reloadDNS manually any more as FTL will detect that it needs to reload by itself Signed-off-by: Adam Warner --- gravity.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 5b950b28..e914732a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -910,11 +910,6 @@ chmod g+w "${piholeDir}" "${gravityDBfile}" # Compute numbers to be displayed gravity_ShowCount -# Determine if DNS has been restarted by this instance of gravity -if [[ -z "${dnsWasOffline:-}" ]]; then - "${PIHOLE_COMMAND}" reloaddns -fi - gravity_Cleanup echo "" From 2784b267ec6424ae1235bf4d0a52d07fe7ab8c11 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 16:40:38 +0000 Subject: [PATCH 082/462] pihole command: read values from pihole-FTL instead of setupvars.conf Signed-off-by: Adam Warner --- advanced/Scripts/utils.sh | 4 ++-- pihole | 33 ++++++++++----------------------- 2 files changed, 12 insertions(+), 25 deletions(-) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index c97ad0cf..8bab396a 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -153,7 +153,7 @@ getFTLPID() { # Example getFTLConfigValue dns.piholePTR ####################### getFTLConfigValue(){ - pihole-FTL --config "${1}" + pihole-FTL --config -q "${1}" } ####################### @@ -166,5 +166,5 @@ getFTLConfigValue(){ # setFTLConfigValue dnsmasq.upstreams '[ "8.8.8.8" , "8.8.4.4" ]' ####################### setFTLConfigValue(){ - pihole-FTL --config "${1}" "${2}" + pihole-FTL --config "${1}" "${2}" >/dev/null } diff --git a/pihole b/pihole index 6796acc6..0c91df35 100755 --- a/pihole +++ b/pihole @@ -11,10 +11,9 @@ readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" -# setupVars and PI_HOLE_BIN_DIR are not readonly here because in some functions (checkout), +# PI_HOLE_BIN_DIR is not readonly here because in some functions (checkout), # they might get set again when the installer is sourced. This causes an # error due to modifying a readonly variable. -setupVars="/etc/pihole/setupVars.conf" PI_HOLE_BIN_DIR="/usr/local/bin" readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" @@ -193,7 +192,7 @@ Time: elif [[ "${1}" == "0" ]]; then # Disable Pi-hole - if grep -cq "BLOCKING_ENABLED=false" "${setupVars}"; then + if ! getFTLConfigValue dns.blocking.active; then echo -e " ${INFO} Blocking already disabled, nothing to do" exit 0 fi @@ -233,19 +232,19 @@ Time: fi local str="Pi-hole Disabled" - addOrEditKeyValPair "${setupVars}" "BLOCKING_ENABLED" "false" + setFTLConfigValue dns.blocking.active false fi else # Enable Pi-hole killall -q pihole-reenable - if grep -cq "BLOCKING_ENABLED=true" "${setupVars}"; then + if getFTLConfigValue dns.blocking.active; then echo -e " ${INFO} Blocking already enabled, nothing to do" exit 0 fi echo -e " ${INFO} Enabling blocking" local str="Pi-hole Enabled" - addOrEditKeyValPair "${setupVars}" "BLOCKING_ENABLED" "true" + setFTLConfigValue dns.blocking.active true fi restartDNS reload-lists @@ -267,8 +266,7 @@ Options: exit 0 elif [[ "${1}" == "off" ]]; then # Disable logging - removeKey /etc/dnsmasq.d/01-pihole.conf "log-queries" - addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "false" + setFTLConfigValue dns.queryLogging false if [[ "${2}" != "noflush" ]]; then # Flush logs "${PI_HOLE_BIN_DIR}"/pihole -f @@ -277,8 +275,7 @@ Options: local str="Logging has been disabled!" elif [[ "${1}" == "on" ]]; then # Enable logging - addKey /etc/dnsmasq.d/01-pihole.conf "log-queries" - addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "true" + setFTLConfigValue dns.queryLogging true echo -e " ${INFO} Enabling logging..." local str="Logging has been enabled!" else @@ -354,26 +351,16 @@ statusFunc() { fi # Determine if Pi-hole's blocking is enabled - if grep -q "BLOCKING_ENABLED=false" /etc/pihole/setupVars.conf; then - # A config is commented out - case "${1}" in - "web") echo 0;; - *) echo -e " ${CROSS} Pi-hole blocking is disabled";; - esac - elif grep -q "BLOCKING_ENABLED=true" /etc/pihole/setupVars.conf; then - # Configs are set + if getFTLConfigValue dns.blocking.active; then case "${1}" in "web") echo "$port";; *) echo -e " ${TICK} Pi-hole blocking is enabled";; esac else - # No configs were found case "${1}" in - "web") echo -2;; - *) echo -e " ${INFO} Pi-hole blocking will be enabled";; + "web") echo 0;; + *) echo -e " ${CROSS} Pi-hole blocking is disabled";; esac - # Enable blocking - "${PI_HOLE_BIN_DIR}"/pihole enable fi exit 0 } From cd17040f959551fbc3250a4565e5fe547f595073 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 17:20:52 +0000 Subject: [PATCH 083/462] setupVars.conf not existing is no longer a critical error Signed-off-by: Adam Warner --- gravity.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index e914732a..515927e8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -46,10 +46,6 @@ curl_connect_timeout=10 setupVars="${piholeDir}/setupVars.conf" if [[ -f "${setupVars}" ]];then source "${setupVars}" -else - echo -e " ${COL_LIGHT_RED}Installation Failure: ${setupVars} does not exist! ${COL_NC} - Please run 'pihole -r', and choose the 'reconfigure' option to fix." - exit 1 fi # Set up tmp dir variable in case it's not configured From 0e8f285f4f670d07be8c8b40d15ebdad3c4afa75 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 18:39:48 +0000 Subject: [PATCH 084/462] Always assume that the web interface will be installed. We could revist this again in future if we decide that there should still be a choice to install the interface or not Signed-off-by: Adam Warner --- advanced/Scripts/piholeCheckout.sh | 27 +++---- advanced/Scripts/piholeDebug.sh | 15 +--- advanced/Scripts/update.sh | 24 +++--- advanced/Scripts/updatecheck.sh | 28 +++---- advanced/Scripts/version.sh | 24 ++---- automated install/basic-install.sh | 115 ++++++++--------------------- automated install/uninstall.sh | 4 - test/test_any_automated_install.py | 1 - 8 files changed, 71 insertions(+), 167 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index cf57800c..5a2eebd7 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -61,12 +61,11 @@ checkout() { echo -e " Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}" exit 1; fi - if [[ "${INSTALL_WEB_INTERFACE}" == "true" ]]; then - if ! is_repo "${webInterfaceDir}" ; then - echo -e " ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!" - echo -e " Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}" - exit 1; - fi + + if ! is_repo "${webInterfaceDir}" ; then + echo -e " ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!" + echo -e " Please re-run install script from https://github.com/pi-hole/pi-hole${COL_NC}" + exit 1; fi if [[ -z "${1}" ]]; then @@ -85,11 +84,9 @@ checkout() { echo "" echo -e " ${INFO} Pi-hole Core" fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "development" || { echo " ${CROSS} Unable to pull Core development branch"; exit 1; } - if [[ "${INSTALL_WEB_INTERFACE}" == "true" ]]; then - echo "" - echo -e " ${INFO} Web interface" - fetch_checkout_pull_branch "${webInterfaceDir}" "devel" || { echo " ${CROSS} Unable to pull Web development branch"; exit 1; } - fi + echo "" + echo -e " ${INFO} Web interface" + fetch_checkout_pull_branch "${webInterfaceDir}" "devel" || { echo " ${CROSS} Unable to pull Web development branch"; exit 1; } #echo -e " ${TICK} Pi-hole Core" local path @@ -101,10 +98,8 @@ checkout() { echo -e " ${INFO} Shortcut \"master\" detected - checking out master branches..." echo -e " ${INFO} Pi-hole core" fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "master" || { echo " ${CROSS} Unable to pull Core master branch"; exit 1; } - if [[ ${INSTALL_WEB_INTERFACE} == "true" ]]; then - echo -e " ${INFO} Web interface" - fetch_checkout_pull_branch "${webInterfaceDir}" "master" || { echo " ${CROSS} Unable to pull Web master branch"; exit 1; } - fi + echo -e " ${INFO} Web interface" + fetch_checkout_pull_branch "${webInterfaceDir}" "master" || { echo " ${CROSS} Unable to pull Web master branch"; exit 1; } #echo -e " ${TICK} Web Interface" local path path="master/${binary}" @@ -137,7 +132,7 @@ checkout() { exit 1 fi checkout_pull_branch "${PI_HOLE_FILES_DIR}" "${2}" - elif [[ "${1}" == "web" ]] && [[ "${INSTALL_WEB_INTERFACE}" == "true" ]] ; then + elif [[ "${1}" == "web" ]] ; then str="Fetching branches from ${webInterfaceGitUrl}" echo -ne " ${INFO} $str" if ! fully_fetch_repo "${webInterfaceDir}" ; then diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 9f895aab..e7c383b5 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -291,17 +291,10 @@ compare_local_version_to_git_version() { return 1 fi else - # There is no git directory so check if the web interface was disabled - local setup_vars_web_interface - setup_vars_web_interface=$(< ${PIHOLE_SETUP_VARS_FILE} grep ^INSTALL_WEB_INTERFACE | cut -d '=' -f2) - if [[ "${pihole_component}" == "Web" ]] && [[ "${setup_vars_web_interface}" == "false" ]]; then - log_write "${INFO} ${pihole_component}: Disabled in setupVars.conf via INSTALL_WEB_INTERFACE=false" - else - # Return an error message - log_write "${COL_RED}Directory ${git_dir} doesn't exist${COL_NC}" - # and exit with a non zero code - return 1 - fi + # Return an error message + log_write "${COL_RED}Directory ${git_dir} doesn't exist${COL_NC}" + # and exit with a non zero code + return 1 fi } diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index c41c9232..6185d94a 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -128,20 +128,18 @@ main() { echo -e " ${INFO} Pi-hole Core:\\t${COL_LIGHT_GREEN}up to date${COL_NC}" fi - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - if ! is_repo "${ADMIN_INTERFACE_DIR}" ; then - echo -e "\\n ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!" - echo -e " Please re-run install script from https://pi-hole.net${COL_NC}" - exit 1; - fi + if ! is_repo "${ADMIN_INTERFACE_DIR}" ; then + echo -e "\\n ${COL_LIGHT_RED}Error: Web Admin repo is missing from system!" + echo -e " Please re-run install script from https://pi-hole.net${COL_NC}" + exit 1; + fi - if GitCheckUpdateAvail "${ADMIN_INTERFACE_DIR}" ; then - web_update=true - echo -e " ${INFO} Web Interface:\\t${COL_YELLOW}update available${COL_NC}" - else - web_update=false - echo -e " ${INFO} Web Interface:\\t${COL_LIGHT_GREEN}up to date${COL_NC}" - fi + if GitCheckUpdateAvail "${ADMIN_INTERFACE_DIR}" ; then + web_update=true + echo -e " ${INFO} Web Interface:\\t${COL_YELLOW}update available${COL_NC}" + else + web_update=false + echo -e " ${INFO} Web Interface:\\t${COL_LIGHT_GREEN}up to date${COL_NC}" fi local funcOutput diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 0cc65218..7d7103d2 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -34,10 +34,6 @@ function get_remote_hash(){ git ls-remote "https://github.com/pi-hole/${1}" --tags "${2}" | awk '{print substr($0, 0,8);}' || return 1 } -# Source the setupvars config file -# shellcheck disable=SC1091 -. /etc/pihole/setupVars.conf - # Source the utils file for addOrEditKeyValPair() # shellcheck disable=SC1091 . /opt/pihole/utils.sh @@ -86,24 +82,20 @@ addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_CORE_HASH" "${GITHUB_CORE_HASH}" # get Web versions -if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then +WEB_VERSION="$(get_local_version /var/www/html/admin)" +addOrEditKeyValPair "${VERSION_FILE}" "WEB_VERSION" "${WEB_VERSION}" - WEB_VERSION="$(get_local_version /var/www/html/admin)" - addOrEditKeyValPair "${VERSION_FILE}" "WEB_VERSION" "${WEB_VERSION}" +WEB_BRANCH="$(get_local_branch /var/www/html/admin)" +addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" - WEB_BRANCH="$(get_local_branch /var/www/html/admin)" - addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" +WEB_HASH="$(get_local_hash /var/www/html/admin)" +addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" - WEB_HASH="$(get_local_hash /var/www/html/admin)" - addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" +GITHUB_WEB_VERSION="$(get_remote_version AdminLTE)" +addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" - GITHUB_WEB_VERSION="$(get_remote_version AdminLTE)" - addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" - - GITHUB_WEB_HASH="$(get_remote_hash AdminLTE "${WEB_BRANCH}")" - addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_HASH" "${GITHUB_WEB_HASH}" - -fi +GITHUB_WEB_HASH="$(get_remote_hash AdminLTE "${WEB_BRANCH}")" +addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_HASH" "${GITHUB_WEB_HASH}" # get FTL versions diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index 946c69fe..af86b045 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -8,10 +8,6 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -# Source the setupvars config file -# shellcheck disable=SC1091 -. /etc/pihole/setupVars.conf - # Source the versions file poupulated by updatechecker.sh cachedVersions="/etc/pihole/versions" @@ -28,7 +24,7 @@ fi getLocalVersion() { case ${1} in "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_VERSION:=N/A}";; + "AdminLTE" ) echo "${WEB_VERSION:=N/A}";; "FTL" ) echo "${FTL_VERSION:=N/A}";; esac } @@ -36,7 +32,7 @@ getLocalVersion() { getLocalHash() { case ${1} in "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_HASH:=N/A}";; + "AdminLTE" ) echo "${WEB_HASH:=N/A}";; "FTL" ) echo "${FTL_HASH:=N/A}";; esac } @@ -44,7 +40,7 @@ getLocalHash() { getRemoteHash(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_HASH:=N/A}";; + "AdminLTE" ) echo "${GITHUB_WEB_HASH:=N/A}";; "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; esac } @@ -52,7 +48,7 @@ getRemoteHash(){ getRemoteVersion(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_VERSION:=N/A}";; + "AdminLTE" ) echo "${GITHUB_WEB_VERSION:=N/A}";; "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; esac } @@ -60,16 +56,12 @@ getRemoteVersion(){ getLocalBranch(){ case ${1} in "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_BRANCH:=N/A}";; + "AdminLTE" ) echo "${WEB_BRANCH:=N/A}";; "FTL" ) echo "${FTL_BRANCH:=N/A}";; esac } versionOutput() { - if [ "$1" = "AdminLTE" ] && [ "${INSTALL_WEB_INTERFACE}" != true ]; then - echo " WebAdmin not installed" - return 1 - fi [ "$2" = "-c" ] || [ "$2" = "--current" ] || [ -z "$2" ] && current=$(getLocalVersion "${1}") && branch=$(getLocalBranch "${1}") [ "$2" = "-l" ] || [ "$2" = "--latest" ] || [ -z "$2" ] && latest=$(getRemoteVersion "${1}") @@ -115,11 +107,7 @@ errorOutput() { defaultOutput() { versionOutput "Pi-hole" "$@" - - if [ "${INSTALL_WEB_INTERFACE}" = true ]; then - versionOutput "AdminLTE" "$@" - fi - + versionOutput "AdminLTE" "$@" versionOutput "FTL" "$@" } diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index bce7aa91..9e5a9963 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -91,7 +91,6 @@ IPV4_ADDRESS=${IPV4_ADDRESS} IPV6_ADDRESS=${IPV6_ADDRESS} # Give settings their default values. These may be changed by prompts later in the script. QUERY_LOGGING=true -INSTALL_WEB_INTERFACE=true WEBPORT=8080 PRIVACY_LEVEL=0 CACHE_SIZE=10000 @@ -1048,44 +1047,6 @@ setPrivacyLevel() { esac } -# Function to ask the user if they want to install the dashboard -setAdminFlag() { - # Similar to the logging function, ask what the user wants - dialog --no-shadow --keep-tite \ - --backtitle "Pihole Installation" \ - --title "Admin Web Interface" \ - --yesno "\\n\\nDo you want to install the Admin Web Interface?" \ - "${r}" "${c}" && result=0 || result=$? - - case ${result} in - "${DIALOG_OK}") - # If they chose yes, - printf " %b Installing Admin Web Interface\\n" "${INFO}" - # Set the flag to install the web interface - INSTALL_WEB_INTERFACE=true - - # Web port TODO: Below whiptail copy pasted from a previous go at this. needs converting to dialog - # Ask for the IPv4 address - WEBPORT=$(whiptail --backtitle "Setting web interface port" --title "Web Port" --inputbox "By default, pihole-FTL listens for http traffic on port 8080. If you wish to change the port, you may do so now. You can also do it later by editing /etc/pihole/pihole-FTL.conf" "${r}" "${c}" "${WEBPORT}" 3>&1 1>&2 2>&3) || \ - # Canceling IPv4 settings window - { echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } - printf " %b The Web interface will be accessible on port: %s\\n" "${INFO}" "${WEBPORT}" - - ;; - "${DIALOG_CANCEL}") - # If they chose no, - printf " %b Not installing Admin Web Interface\\n" "${INFO}" - # Set the flag to not install the web interface - INSTALL_WEB_INTERFACE=false - ;; - "${DIALOG_ESC}") - # User pressed - printf " %b Escape pressed, exiting installer at Admin Web Interface choice.%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac -} - # A function to display a list of example blocklists for users to select chooseBlocklists() { # Back up any existing adlist file, on the off chance that it exists. Useful in case of a reconfigure. @@ -1613,7 +1574,6 @@ finalExports() { addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_1" "${PIHOLE_DNS_1}" addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_2" "${PIHOLE_DNS_2}" addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "${QUERY_LOGGING}" - addOrEditKeyValPair "${setupVars}" "INSTALL_WEB_INTERFACE" "${INSTALL_WEB_INTERFACE}" addOrEditKeyValPair "${setupVars}" "CACHE_SIZE" "${CACHE_SIZE}" addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "${DNS_FQDN_REQUIRED:-true}" addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "${DNS_BOGUS_PRIV:-true}" @@ -1775,11 +1735,10 @@ displayFinalMessage() { # Else, inform the user that there is no set password. pwstring="NOT SET" fi - # If the user wants to install the dashboard, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # Store a message in a variable and display it - additional="View the web interface at http://pi.hole/admin:${WEBPORT} or http://${IPV4_ADDRESS%/*}:${WEBPORT}/admin\\n\\nYour Admin Webpage login password is ${pwstring}" - fi + + # Store a message in a variable and display it + additional="View the web interface at http://pi.hole/admin:${WEBPORT} or http://${IPV4_ADDRESS%/*}:${WEBPORT}/admin\\n\\nYour Admin Webpage login password is ${pwstring}" + # Final completion message to user dialog --no-shadow --keep-tite \ @@ -1928,14 +1887,11 @@ clone_or_update_repos() { { printf " %b Unable to reset %s, exiting installer%b\\n" "${COL_LIGHT_RED}" "${PI_HOLE_LOCAL_REPO}" "${COL_NC}"; \ exit 1; \ } - # If the Web interface was installed, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # reset it's repo - resetRepo ${webInterfaceDir} || \ - { printf " %b Unable to reset %s, exiting installer%b\\n" "${COL_LIGHT_RED}" "${webInterfaceDir}" "${COL_NC}"; \ - exit 1; \ - } - fi + # Reset the Web repo + resetRepo ${webInterfaceDir} || \ + { printf " %b Unable to reset %s, exiting installer%b\\n" "${COL_LIGHT_RED}" "${webInterfaceDir}" "${COL_NC}"; \ + exit 1; \ + } # Otherwise, a repair is happening else # so get git files for Core @@ -1943,14 +1899,11 @@ clone_or_update_repos() { { printf " %b Unable to clone %s into %s, unable to continue%b\\n" "${COL_LIGHT_RED}" "${piholeGitUrl}" "${PI_HOLE_LOCAL_REPO}" "${COL_NC}"; \ exit 1; \ } - # If the Web interface was installed, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # get the Web git files - getGitFiles ${webInterfaceDir} ${webInterfaceGitUrl} || \ - { printf " %b Unable to clone %s into ${webInterfaceDir}, exiting installer%b\\n" "${COL_LIGHT_RED}" "${webInterfaceGitUrl}" "${COL_NC}"; \ - exit 1; \ - } - fi + # get the Web git files + getGitFiles ${webInterfaceDir} ${webInterfaceGitUrl} || \ + { printf " %b Unable to clone %s into ${webInterfaceDir}, exiting installer%b\\n" "${COL_LIGHT_RED}" "${webInterfaceGitUrl}" "${COL_NC}"; \ + exit 1; \ + } fi } @@ -2253,7 +2206,7 @@ main() { # when run via curl piping if [[ "$0" == "bash" ]]; then # Download the install script and run it with admin rights - exec curl -sSL https://raw.githubusercontent.com/pi-hole/pi-hole/master/automated%20install/basic-install.sh | sudo bash "$@" + exec curl -sSL https://install.pi-hole.net | sudo bash "$@" else # when run via calling local bash script exec sudo bash "$0" "$@" @@ -2319,8 +2272,6 @@ main() { setDNS # Give the user a choice of blocklists to include in their install. Or not. chooseBlocklists - # Let the user decide if they want the web interface to be installed automatically - setAdminFlag # Let the user decide if they want query logging enabled... setLogging # Let the user decide the FTL privacy level @@ -2373,17 +2324,13 @@ main() { # Copy the temp log file into final log location for storage copy_to_install_log - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # Add password to web UI if there is none - pw="" - # If no password is set, - if [[ $(grep 'WEBPASSWORD' -c "${setupVars}") == 0 ]] ; then - # generate a random password - pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8) - # shellcheck disable=SC1091 - . /opt/pihole/webpage.sh - echo "WEBPASSWORD=$(HashPassword "${pw}")" >> "${setupVars}" - fi + # Add password to web UI if there is none + pw="" + # If no password is set, + if [[ $(pihole-FTL --config webserver.api.pwhash) == "${pw}" ]] ; then + # generate a random password + pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8) + pihole -a -p "${pw}" fi # Check for and disable systemd-resolved-DNSStubListener before reloading resolved @@ -2441,21 +2388,17 @@ main() { displayFinalMessage "${pw}" fi - # If the Web interface was installed, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - # If there is a password, - if (( ${#pw} > 0 )) ; then - # display the password - printf " %b Web Interface password: %b%s%b\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${pw}" "${COL_NC}" - printf " %b This can be changed using 'pihole -a -p'\\n\\n" "${INFO}" - fi + # If there is a password + if (( ${#pw} > 0 )) ; then + # display the password + printf " %b Web Interface password: %b%s%b\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${pw}" "${COL_NC}" + printf " %b This can be changed using 'pihole -a -p'\\n\\n" "${INFO}" fi if [[ "${useUpdateVars}" == false ]]; then # If the Web interface was installed, - if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then - printf " %b View the web interface at http://pi.hole:${WEBPORT}/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}:${WEBPORT}" - fi + printf " %b View the web interface at http://pi.hole:${WEBPORT}/admin or http://%s/admin\\n\\n" "${INFO}" "${IPV4_ADDRESS%/*}:${WEBPORT}" + # Explain to the user how to use Pi-hole as their DNS server printf " %b You may now configure your devices to use the Pi-hole as their DNS server\\n" "${INFO}" [[ -n "${IPV4_ADDRESS%/*}" ]] && printf " %b Pi-hole DNS (IPv4): %s\\n" "${INFO}" "${IPV4_ADDRESS%/*}" diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 7a1a290d..8cffae20 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -46,10 +46,6 @@ package_manager_detect # Uninstall packages used by the Pi-hole DEPS=("${INSTALLER_DEPS[@]}" "${PIHOLE_DEPS[@]}" "${OS_CHECK_DEPS[@]}") -if [[ "${INSTALL_WEB_SERVER}" == true ]]; then - # Install the Web dependencies - DEPS+=("${PIHOLE_WEB_DEPS[@]}") -fi # Compatibility if [ -x "$(command -v apt-get)" ]; then diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 66d00814..42c94e63 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -120,7 +120,6 @@ def test_installPihole_fresh_install_readableFiles(host): setup_var_file = "cat < /etc/pihole/setupVars.conf\n" for k, v in SETUPVARS.items(): setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "INSTALL_WEB_INTERFACE=true\n" setup_var_file += "EOF\n" host.run(setup_var_file) install = host.run( From 7cbe713873d38eb4b4952ea7e24a59f4c0084ed7 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 18:59:38 +0000 Subject: [PATCH 085/462] REVISIT: Don't rely on existence of setupVars.conf Signed-off-by: Adam Warner --- advanced/Scripts/update.sh | 3 -- advanced/Scripts/utils.sh | 4 +- automated install/basic-install.sh | 65 ++++++------------------------ test/conftest.py | 7 ---- test/test_any_automated_install.py | 47 --------------------- 5 files changed, 15 insertions(+), 111 deletions(-) diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 6185d94a..b6153293 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -104,9 +104,6 @@ main() { web_update=false FTL_update=false - # shellcheck disable=1090,2154 - source "${setupVars}" - # Install packages used by this installation script (necessary if users have removed e.g. git from their systems) package_manager_detect install_dependent_packages "${INSTALLER_DEPS[@]}" diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 8bab396a..205edd90 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -162,8 +162,8 @@ getFTLConfigValue(){ # Takes two arguments: key and value # Example setFTLConfigValue dns.piholePTR PI.HOLE # -# Note, for complex values such as dnsmasq.upstreams, you should wrap the value in single quotes: -# setFTLConfigValue dnsmasq.upstreams '[ "8.8.8.8" , "8.8.4.4" ]' +# Note, for complex values such as dns.upstreams, you should wrap the value in single quotes: +# setFTLConfigValue dns.upstreams '[ "8.8.8.8" , "8.8.4.4" ]' ####################### setFTLConfigValue(){ pihole-FTL --config "${1}" "${2}" >/dev/null diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9e5a9963..984fe0c0 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -56,8 +56,6 @@ EOM # Location for final installation log storage installLogLoc="/etc/pihole/install.log" -# This is an important file as it contains information specific to the machine it's being installed on -setupVars="/etc/pihole/setupVars.conf" # This is a file used for the colorized output coltable="/opt/pihole/COL_TABLE" @@ -1566,29 +1564,7 @@ create_pihole_user() { fi } -# This function saves any changes to the setup variables into the setupvars.conf file for future runs finalExports() { - # set or update the variables in the file - - addOrEditKeyValPair "${setupVars}" "PIHOLE_INTERFACE" "${PIHOLE_INTERFACE}" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_1" "${PIHOLE_DNS_1}" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_2" "${PIHOLE_DNS_2}" - addOrEditKeyValPair "${setupVars}" "QUERY_LOGGING" "${QUERY_LOGGING}" - addOrEditKeyValPair "${setupVars}" "CACHE_SIZE" "${CACHE_SIZE}" - addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "${DNS_FQDN_REQUIRED:-true}" - addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "${DNS_BOGUS_PRIV:-true}" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "${DNSMASQ_LISTENING:-local}" - - chmod 644 "${setupVars}" - - # Set the privacy level - addOrEditKeyValPair "${FTL_CONFIG_FILE}" "PRIVACYLEVEL" "${PRIVACY_LEVEL}" - - # Set the web port - addOrEditKeyValPair "${FTL_CONFIG_FILE}" "WEBPORT" "${WEBPORT}" - - # Bring in the current settings and the functions to manipulate them - source "${setupVars}" # shellcheck source=advanced/Scripts/webpage.sh source "${PI_HOLE_LOCAL_REPO}/advanced/Scripts/webpage.sh" @@ -1672,7 +1648,6 @@ installPihole() { # install a man page entry for pihole install_manpage - # Update setupvars.conf with any variables that may or may not have been changed during the install finalExports } @@ -1724,11 +1699,12 @@ checkSelinux() { # Installation complete message with instructions for the user displayFinalMessage() { + # TODO: COME BACK TO THIS, WHAT IS GOING ON? # If the number of arguments is > 0, if [[ "${#1}" -gt 0 ]] ; then # set the password to the first argument. pwstring="$1" - elif [[ $(grep 'WEBPASSWORD' -c "${setupVars}") -gt 0 ]]; then + elif [[ $(pihole-FTL --config webserver.api.pwhash) == '""' ]] ; then # Else if the password exists from previous setup, we'll load it later pwstring="unchanged" else @@ -2242,19 +2218,16 @@ main() { printf " %b Checking for / installing Required dependencies for this install script...\\n" "${INFO}" install_dependent_packages "${INSTALLER_DEPS[@]}" - # If the setup variable file exists, - if [[ -f "${setupVars}" ]]; then - # if it's running unattended, - if [[ "${runUnattended}" == true ]]; then - printf " %b Performing unattended setup, no dialogs will be displayed\\n" "${INFO}" - # Use the setup variables - useUpdateVars=true - # also disable debconf-apt-progress dialogs - export DEBIAN_FRONTEND="noninteractive" - else - # If running attended, show the available options (repair/reconfigure) - update_dialogs - fi + # if it's running unattended, + if [[ "${runUnattended}" == true ]]; then + printf " %b Performing unattended setup, no dialogs will be displayed\\n" "${INFO}" + # Use the setup variables + useUpdateVars=true + # also disable debconf-apt-progress dialogs + export DEBIAN_FRONTEND="noninteractive" + else + # If running attended, show the available options (repair/reconfigure) + update_dialogs fi if [[ "${useUpdateVars}" == false ]]; then @@ -2279,18 +2252,6 @@ main() { else # Setup adlist file if not exists installDefaultBlocklists - - # Source ${setupVars} to use predefined user variables in the functions - source "${setupVars}" - - # Get the privacy level if it exists (default is 0) - if [[ -f "${FTL_CONFIG_FILE}" ]]; then - # get the value from $FTL_CONFIG_FILE (and ignoring all commented lines) - PRIVACY_LEVEL=$(sed -e '/^[[:blank:]]*#/d' "${FTL_CONFIG_FILE}" | grep "PRIVACYLEVEL" | awk -F "=" 'NR==1{printf$2}') - - # If no setting was found, default to 0 - PRIVACY_LEVEL="${PRIVACY_LEVEL:-0}" - fi fi # Download or update the scripts by updating the appropriate git repos clone_or_update_repos @@ -2327,7 +2288,7 @@ main() { # Add password to web UI if there is none pw="" # If no password is set, - if [[ $(pihole-FTL --config webserver.api.pwhash) == "${pw}" ]] ; then + if [[ $(pihole-FTL --config webserver.api.pwhash) == '""' ]] ; then # generate a random password pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8) pihole -a -p "${pw}" diff --git a/test/conftest.py b/test/conftest.py index e395ec27..164e8de5 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -4,13 +4,6 @@ import testinfra.backend.docker import subprocess from textwrap import dedent - -SETUPVARS = { - "PIHOLE_INTERFACE": "eth99", - "PIHOLE_DNS_1": "4.2.2.1", - "PIHOLE_DNS_2": "4.2.2.2", -} - IMAGE = "pytest_pihole:test_container" tick_box = "[\x1b[1;32m\u2713\x1b[0m]" diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 42c94e63..45cb01c5 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -2,7 +2,6 @@ import pytest from textwrap import dedent import re from .conftest import ( - SETUPVARS, tick_box, info_box, cross_box, @@ -32,42 +31,6 @@ def test_supported_package_manager(host): # assert package_manager_detect.rc == 1 -def test_setupVars_are_sourced_to_global_scope(host): - """ - currently update_dialogs sources setupVars with a dot, - then various other functions use the variables. - This confirms the sourced variables are in scope between functions - """ - setup_var_file = "cat < /etc/pihole/setupVars.conf\n" - for k, v in SETUPVARS.items(): - setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "EOF\n" - host.run(setup_var_file) - - script = dedent( - """\ - set -e - printSetupVars() { - # Currently debug test function only - echo "Outputting sourced variables" - echo "PIHOLE_INTERFACE=${PIHOLE_INTERFACE}" - echo "PIHOLE_DNS_1=${PIHOLE_DNS_1}" - echo "PIHOLE_DNS_2=${PIHOLE_DNS_2}" - } - update_dialogs() { - . /etc/pihole/setupVars.conf - } - update_dialogs - printSetupVars - """ - ) - - output = run_script(host, script).stdout - - for k, v in SETUPVARS.items(): - assert "{}={}".format(k, v) in output - - def test_selinux_not_detected(host): """ confirms installer continues when SELinux configuration file does not exist @@ -116,12 +79,6 @@ def test_installPihole_fresh_install_readableFiles(host): host.run("command -v apt-get > /dev/null && apt-get install -qq man") host.run("command -v dnf > /dev/null && dnf install -y man") host.run("command -v yum > /dev/null && yum install -y man") - # create configuration file - setup_var_file = "cat < /etc/pihole/setupVars.conf\n" - for k, v in SETUPVARS.items(): - setup_var_file += "{}={}\n".format(k, v) - setup_var_file += "EOF\n" - host.run(setup_var_file) install = host.run( """ export TERM=xterm @@ -185,10 +142,6 @@ def test_installPihole_fresh_install_readableFiles(host): check_FTLconf = test_cmd.format("w", "/etc/pihole/pihole-FTL.conf", piholeuser) actual_rc = host.run(check_FTLconf).rc assert exit_status_success == actual_rc - # readable setupVars.conf - check_setup = test_cmd.format("r", "/etc/pihole/setupVars.conf", piholeuser) - actual_rc = host.run(check_setup).rc - assert exit_status_success == actual_rc # check readable and executable /etc/init.d/pihole-FTL check_init = test_cmd.format("x", "/etc/init.d/pihole-FTL", piholeuser) actual_rc = host.run(check_init).rc From 6b1c8a7fff92ae08a2b42d2e827249e6e2d345ca Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 5 Feb 2023 21:30:31 +0000 Subject: [PATCH 086/462] @DL6ER changed the config key names... Remove superfluous test Signed-off-by: Adam Warner --- test/test_any_utils.py | 31 ++----------------------------- 1 file changed, 2 insertions(+), 29 deletions(-) diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 741b1127..8dc3f1a5 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -172,33 +172,6 @@ def test_getFTLPIDFile_and_getFTLPID_custom(host): assert expected_stdout == output.stdout -def test_setFTLConfigValue_getFTLConfigValue(host): - """ - Confirms setFTLConfigValue works - Requires FTL to be installed, so we do that first (taken from test_FTL_binary_installed_and_responsive_no_errors) - """ - host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - echo "new/http" > /etc/pihole/ftlbranch - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - - output = host.run( - """ - source /opt/pihole/utils.sh - setFTLConfigValue "dnsmasq.upstreams" '["1.1.1.1"]' - """ - ) - - assert '[ "1.1.1.1" ]' in output.stdout - - def test_getFTLConfigValue_getFTLConfigValue(host): """ Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) @@ -219,8 +192,8 @@ def test_getFTLConfigValue_getFTLConfigValue(host): output = host.run( """ source /opt/pihole/utils.sh - setFTLConfigValue "dnsmasq.upstreams" '["9.9.9.9"]' > /dev/null - getFTLConfigValue "dnsmasq.upstreams" + setFTLConfigValue "dns.upstreams" '["9.9.9.9"]' > /dev/null + getFTLConfigValue "dns.upstreams" """ ) From a91eb48d48f18026a87067e5ba0ce86a4d91a125 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 11 Feb 2023 12:34:12 +0000 Subject: [PATCH 087/462] Remove some code from gravity that gets values from setupVars.conf / pihole-FTL.conf - use getFTLConfigValue from utils.sh instead Signed-off-by: Adam Warner --- gravity.sh | 39 ++++++++++++--------------------------- 1 file changed, 12 insertions(+), 27 deletions(-) diff --git a/gravity.sh b/gravity.sh index 515927e8..66a90449 100755 --- a/gravity.sh +++ b/gravity.sh @@ -13,10 +13,17 @@ export LC_ALL=C -coltable="/opt/pihole/COL_TABLE" -source "${coltable}" +PI_HOLE_SCRIPT_DIR="/opt/pihole" +# Source utils.sh for GetFTLConfigValue +utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +# shellcheck disable=SC1090 +. "${utilsfile}" + +coltable="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" +# shellcheck disable=SC1090 +. "${coltable}" # shellcheck disable=SC1091 -source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" +. "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" @@ -33,20 +40,13 @@ localList="${piholeDir}/local.list" VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" -gravityDBfile_default="${piholeDir}/gravity.db" -# GRAVITYDB may be overwritten by source pihole-FTL.conf below -GRAVITYDB="${gravityDBfile_default}" +GRAVITYDB=$(getFTLConfigValue files.gravity) gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" domainsExtension="domains" curl_connect_timeout=10 -# Source setupVars from install script -setupVars="${piholeDir}/setupVars.conf" -if [[ -f "${setupVars}" ]];then - source "${setupVars}" -fi # Set up tmp dir variable in case it's not configured : "${GRAVITY_TMPDIR:=/tmp}" @@ -56,12 +56,6 @@ if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then GRAVITY_TMPDIR="/tmp" fi -# Source pihole-FTL from install script -pihole_FTL="${piholeDir}/pihole-FTL.conf" -if [[ -f "${pihole_FTL}" ]]; then - source "${pihole_FTL}" -fi - # Set this only after sourcing pihole-FTL.conf as the gravity database path may # have changed gravityDBfile="${GRAVITYDB}" @@ -69,15 +63,6 @@ gravityTEMPfile="${GRAVITYDB}_temp" gravityDIR="$(dirname -- "${gravityDBfile}")" gravityOLDfile="${gravityDIR}/gravity_old.db" -if [[ -z "${BLOCKINGMODE}" ]] ; then - BLOCKINGMODE="NULL" -fi - -# Determine if superseded pihole.conf exists -if [[ -r "${piholeDir}/pihole.conf" ]]; then - echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}" -fi - # Generate new SQLite3 file from schema template generate_gravity_database() { if ! pihole-FTL sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then @@ -519,7 +504,7 @@ gravity_DownloadBlocklistFromUrl() { str="Status:" echo -ne " ${INFO} ${str} Pending..." blocked=false - case $BLOCKINGMODE in + case $(getFTLConfigValue dns.blocking.mode) in "IP-NODATA-AAAA"|"IP") # Get IP address of this domain ip="$(dig "${domain}" +short)" From c0e352094da78f1d888e34bbc6dc511e3e069bb0 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 11 Feb 2023 12:51:55 +0000 Subject: [PATCH 088/462] REVISIT: Remove some references to setupVars (some files can probably go entirely...) Signed-off-by: Adam Warner --- advanced/Scripts/chronometer.sh | 8 -------- advanced/Scripts/piholeCheckout.sh | 3 --- advanced/Scripts/piholeDebug.sh | 2 +- advanced/Scripts/utils.sh | 1 + advanced/Scripts/webpage.sh | 8 ++++---- 5 files changed, 6 insertions(+), 16 deletions(-) diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index d69a56d3..99cff230 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -210,11 +210,6 @@ get_init_stats() { else temp_file="" fi - - # Test existence of setupVars config - if [[ -f "/etc/pihole/setupVars.conf" ]]; then - setupVars="/etc/pihole/setupVars.conf" - fi } get_sys_stats() { @@ -226,9 +221,6 @@ get_sys_stats() { # Update every 12 refreshes (Def: every 60s) count=$((count+1)) if [[ "$count" == "1" ]] || (( "$count" % 12 == 0 )); then - # Do not source setupVars if file does not exist - [[ -n "$setupVars" ]] && source "$setupVars" - mapfile -t ph_ver_raw < <(pihole -v -c 2> /dev/null | sed -n 's/^.* v/v/p') if [[ -n "${ph_ver_raw[0]}" ]]; then ph_core_ver="${ph_ver_raw[0]}" diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 5a2eebd7..39d39b1c 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -16,15 +16,12 @@ source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" # webInterfaceDir set in basic-install.sh # piholeGitURL set in basic-install.sh # is_repo() sourced from basic-install.sh -# setupVars set in basic-install.sh # check_download_exists sourced from basic-install.sh # fully_fetch_repo sourced from basic-install.sh # get_available_branches sourced from basic-install.sh # fetch_checkout_pull_branch sourced from basic-install.sh # checkout_pull_branch sourced from basic-install.sh -source "${setupVars}" - warning1() { echo " Please note that changing branches severely alters your Pi-hole subsystems" echo " Features that work on the master branch, may not on a development branch" diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index e7c383b5..ef1d8558 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -77,7 +77,6 @@ PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" -PIHOLE_SETUP_VARS_FILE="${PIHOLE_DIRECTORY}/setupVars.conf" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole-FTL.conf" PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" @@ -1465,6 +1464,7 @@ upload_to_tricorder() { # Run through all the functions we made make_temporary_log initialize_debug +# TODO: Address the reliance on setupVars.conf here. Should debug read pihole.toml directly, or rely on pihole-FTL --config? # setupVars.conf needs to be sourced before the networking so the values are # available to the other functions source_setup_variables diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 205edd90..8f33d678 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -25,6 +25,7 @@ # # Example usage: # addOrEditKeyValPair "/etc/pihole/setupVars.conf" "BLOCKING_ENABLED" "true" +# TODO: We miight not actually need this function in v6 ####################### addOrEditKeyValPair() { local file="${1}" diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 53f36a20..8fd0253b 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -12,6 +12,8 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. +# TODO - this entire file might be able to be removed in v6 + readonly dnsmasqconfig="/etc/dnsmasq.d/01-pihole.conf" readonly dhcpconfig="/etc/dnsmasq.d/02-pihole-dhcp.conf" readonly FTLconf="/etc/pihole/pihole-FTL.conf" @@ -175,10 +177,8 @@ SetWebPassword() { fi if [ "${PASSWORD}" == "${CONFIRM}" ] ; then - # We do not wrap this in brackets, otherwise BASH will expand any appropriate syntax - hash=$(HashPassword "$PASSWORD") - # Save hash to file - setFTLConfigValue "webserver.api.pwhash" "${hash}" >/dev/null + # pihole-FTL will automatically hash the password + setFTLConfigValue "webserver.api.password" "${PASSWORD}" >/dev/null echo -e " ${TICK} New password set" else echo -e " ${CROSS} Passwords don't match. Your password has not been changed" From ea748822ef5d6e0962a25005b32ea5357a85614f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 12 Feb 2023 17:39:37 +0000 Subject: [PATCH 089/462] REVISIT: I don't _think_ this line is nessacery. Why should gravity restart FTL if it is offline? It might be offline on purpose. Signed-off-by: Adam Warner --- gravity.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 66a90449..3b08ff2b 100755 --- a/gravity.sh +++ b/gravity.sh @@ -720,11 +720,11 @@ gravity_Cleanup() { echo -e "${OVER} ${TICK} ${str}" - # Only restart DNS service if offline - if ! pgrep pihole-FTL &> /dev/null; then - "${PIHOLE_COMMAND}" restartdns - dnsWasOffline=true - fi + # # Only restart DNS service if offline + # if ! pgrep pihole-FTL &> /dev/null; then + # "${PIHOLE_COMMAND}" restartdns + # dnsWasOffline=true + # fi # Print Pi-hole status if an error occurred if [[ -n "${error}" ]]; then @@ -894,4 +894,4 @@ gravity_ShowCount gravity_Cleanup echo "" -"${PIHOLE_COMMAND}" status +# "${PIHOLE_COMMAND}" status From 44bfb8ebf095caa2f68d68e89e73fb194c4034c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 24 Feb 2023 21:11:19 +0100 Subject: [PATCH 090/462] Remove the ability to reboot/shutdown via webpage.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 8fd0253b..4c321ec2 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -401,14 +401,6 @@ SetExcludeClients() { addOrEditKeyValPair "${setupVars}" "API_EXCLUDE_CLIENTS" "${args[2]}" } -Poweroff(){ - nohup bash -c "sleep 5; poweroff" &> /dev/null /dev/null Date: Fri, 24 Feb 2023 21:31:35 +0100 Subject: [PATCH 091/462] Remove getFTLAPIPort() function and fix pihole status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/chronometer.sh | 2 +- advanced/Scripts/utils.sh | 23 ----------------- pihole | 7 +++-- test/test_any_utils.py | 46 --------------------------------- 4 files changed, 4 insertions(+), 74 deletions(-) diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index 99cff230..49de6efd 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -16,7 +16,7 @@ pihole-FTL() { local ftl_port LINE # shellcheck disable=SC1091 . /opt/pihole/utils.sh - ftl_port=$(getFTLAPIPort) + ftl_port=$(getFTLConfigValue dns.port) if [[ -n "$ftl_port" ]]; then # Open connection to FTL exec 3<>"/dev/tcp/127.0.0.1/$ftl_port" diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 8f33d678..3f9b7031 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -81,29 +81,6 @@ removeKey() { sed -i "/^${key}/d" "${file}" } - -####################### -# returns FTL's current telnet API port based on the setting in /etc/pihole-FTL.conf -######################## -getFTLAPIPort(){ - local FTLCONFFILE="/etc/pihole/pihole-FTL.conf" - local DEFAULT_FTL_PORT=4711 - local ftl_api_port - - if [ -s "$FTLCONFFILE" ]; then - # if FTLPORT is not set in pihole-FTL.conf, use the default port - ftl_api_port="$({ grep '^FTLPORT=' "${FTLCONFFILE}" || echo "${DEFAULT_FTL_PORT}"; } | cut -d'=' -f2-)" - # Exploit prevention: set the port to the default port if there is malicious (non-numeric) - # content set in pihole-FTL.conf - expr "${ftl_api_port}" : "[^[:digit:]]" > /dev/null && ftl_api_port="${DEFAULT_FTL_PORT}" - else - # if there is no pihole-FTL.conf, use the default port - ftl_api_port="${DEFAULT_FTL_PORT}" - fi - - echo "${ftl_api_port}" -} - ####################### # returns path of FTL's PID file ####################### diff --git a/pihole b/pihole index 0c91df35..0be995e4 100755 --- a/pihole +++ b/pihole @@ -320,13 +320,12 @@ analyze_ports() { statusFunc() { # Determine if there is pihole-FTL service is listening - local pid port ftl_api_port ftl_pid_file + local pid port ftl_pid_file ftl_pid_file="$(getFTLPIDFile)" pid="$(getFTLPID ${ftl_pid_file})" - ftl_api_port="$(getFTLAPIPort)" if [[ "$pid" -eq "-1" ]]; then case "${1}" in "web") echo "-1";; @@ -334,8 +333,8 @@ statusFunc() { esac return 0 else - #get the DNS port pihole-FTL is listening on by using FTL's telnet API - port="$(echo ">dns-port >quit" | nc 127.0.0.1 "$ftl_api_port")" + # get the DNS port pihole-FTL is listening on + port="$(getFTLConfigValue dns.port)" if [[ "${port}" == "0" ]]; then case "${1}" in "web") echo "-1";; diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 8dc3f1a5..efab3760 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -82,52 +82,6 @@ def test_key_removal_works(host): assert expected_stdout == output.stdout -def test_getFTLAPIPort_default(host): - """Confirms getFTLAPIPort returns the default API port""" - output = host.run( - """ - source /opt/pihole/utils.sh - getFTLAPIPort - """ - ) - expected_stdout = "4711\n" - assert expected_stdout == output.stdout - - -def test_getFTLAPIPort_custom(host): - """Confirms getFTLAPIPort returns a custom API port""" - host.run( - """ - echo "FTLPORT=1234" > /etc/pihole/pihole-FTL.conf - """ - ) - output = host.run( - """ - source /opt/pihole/utils.sh - getFTLAPIPort - """ - ) - expected_stdout = "1234\n" - assert expected_stdout == output.stdout - - -def test_getFTLAPIPort_malicious(host): - """Confirms getFTLAPIPort returns 4711 if the setting in pihole-FTL.conf contains non-digits""" - host.run( - """ - echo "FTLPORT=*$ssdfsd" > /etc/pihole/pihole-FTL.conf - """ - ) - output = host.run( - """ - source /opt/pihole/utils.sh - getFTLAPIPort - """ - ) - expected_stdout = "4711\n" - assert expected_stdout == output.stdout - - def test_getFTLPIDFile_default(host): """Confirms getFTLPIDFile returns the default PID file path""" output = host.run( From 783f9e556950fe06ab76a309a19d38e40c3f015c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 May 2023 10:56:41 +0000 Subject: [PATCH 092/462] Bump actions/setup-python from 4.6.0 to 4.6.1 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.6.0 to 4.6.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.6.0...v4.6.1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2dceff1c..a9f116f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,7 +66,7 @@ jobs: uses: actions/checkout@v3.5.2 - name: Set up Python 3.10 - uses: actions/setup-python@v4.6.0 + uses: actions/setup-python@v4.6.1 with: python-version: "3.10" From 3ae72114c76eb538bfabc4f374b92184d28d2fcc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 May 2023 10:59:32 +0000 Subject: [PATCH 093/462] Bump pytest-testinfra from 8.0.0 to 8.1.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 8.0.0 to 8.1.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/8.0.0...8.1.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 55200286..f11d1253 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ docker-compose == 1.29.2 pytest == 7.3.1 pytest-xdist == 3.3.1 -pytest-testinfra == 8.0.0 +pytest-testinfra == 8.1.0 tox == 4.5.1 From 137e6dc1843c2b9ccfd1e66632f9b476a4d5fd0d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 28 May 2023 08:57:14 +0200 Subject: [PATCH 094/462] Remove all the undocumented now useless stuff from pihole -a Signed-off-by: DL6ER --- advanced/Scripts/webpage.sh | 790 +----------------------------------- 1 file changed, 5 insertions(+), 785 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 4c321ec2..67cbe766 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -43,126 +43,23 @@ fi helpFunc() { echo "Usage: pihole -a [options] Example: pihole -a -p password -Set options for the Admin Console +Set options for the API/Web interface Options: - -p, password Set Admin Console password - -c, celsius Set Celsius as preferred temperature unit - -f, fahrenheit Set Fahrenheit as preferred temperature unit - -k, kelvin Set Kelvin as preferred temperature unit - -h, --help Show this help dialog - -i, interface Specify dnsmasq's interface listening behavior - -l, privacylevel Set privacy level (0 = lowest, 3 = highest) - -t, teleporter Backup configuration as an archive - -t, teleporter myname.tar.gz Backup configuration to archive with name myname.tar.gz as specified" + -p, password Set API/Web interface password + -h, --help Show this help dialog" exit 0 } -add_setting() { - addOrEditKeyValPair "${setupVars}" "${1}" "${2}" -} - -delete_setting() { - removeKey "${setupVars}" "${1}" -} - -change_setting() { - addOrEditKeyValPair "${setupVars}" "${1}" "${2}" -} - -addFTLsetting() { - addOrEditKeyValPair "${FTLconf}" "${1}" "${2}" -} - -deleteFTLsetting() { - removeKey "${FTLconf}" "${1}" -} - -changeFTLsetting() { - pihole-FTL --config "${1}" "${2}" -} - -add_dnsmasq_setting() { - addOrEditKeyValPair "${dnsmasqconfig}" "${1}" "${2}" -} - -delete_dnsmasq_setting() { - removeKey "${dnsmasqconfig}" "${1}" -} - -SetTemperatureUnit() { - addOrEditKeyValPair "${setupVars}" "TEMPERATUREUNIT" "${unit}" - echo -e " ${TICK} Set temperature unit to ${unit}" -} - -HashPassword() { - # Compute password hash twice to avoid rainbow table vulnerability - return=$(echo -n "${1}" | sha256sum | sed 's/\s.*$//') - return=$(echo -n "${return}" | sha256sum | sed 's/\s.*$//') - echo "${return}" -} - -# Check an IP address to see if it is a valid one -valid_ip() { - # Local, named variables - local ip=${1} - local stat=1 - - # Regex matching one IPv4 component, i.e. an integer from 0 to 255. - # See https://tools.ietf.org/html/rfc1340 - local ipv4elem="(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]?|0)"; - # Regex matching an optional port (starting with '#') range of 1-65536 - local portelem="(#(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?"; - # Build a full IPv4 regex from the above subexpressions - local regex="^${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}\\.${ipv4elem}${portelem}$" - - # Evaluate the regex, and return the result - [[ $ip =~ ${regex} ]] - - stat=$? - return "${stat}" -} - -valid_ip6() { - local ip=${1} - local stat=1 - - # Regex matching one IPv6 element, i.e. a hex value from 0000 to FFFF - local ipv6elem="[0-9a-fA-F]{1,4}" - # Regex matching an IPv6 CIDR, i.e. 1 to 128 - local v6cidr="(\\/([1-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8])){0,1}" - # Regex matching an optional port (starting with '#') range of 1-65536 - local portelem="(#(6553[0-5]|655[0-2][0-9]|65[0-4][0-9]{2}|6[0-4][0-9]{3}|[1-5][0-9]{4}|[1-9][0-9]{0,3}|0))?"; - # Build a full IPv6 regex from the above subexpressions - local regex="^(((${ipv6elem}))*((:${ipv6elem}))*::((${ipv6elem}))*((:${ipv6elem}))*|((${ipv6elem}))((:${ipv6elem})){7})${v6cidr}${portelem}$" - - # Evaluate the regex, and return the result - [[ ${ip} =~ ${regex} ]] - - stat=$? - return "${stat}" -} - +# TODO: We can probably remove the reliance on this function too, just tell people to pihole-FTL --config webserver.api.password "password" SetWebPassword() { - if [ "${SUDO_USER}" == "www-data" ]; then - echo "Security measure: user www-data is not allowed to change webUI password!" - echo "Exiting" - exit 1 - fi - - if [ "${SUDO_USER}" == "lighttpd" ]; then - echo "Security measure: user lighttpd is not allowed to change webUI password!" - echo "Exiting" - exit 1 - fi - if (( ${#args[2]} > 0 )) ; then readonly PASSWORD="${args[2]}" readonly CONFIRM="${PASSWORD}" else # Prevents a bug if the user presses Ctrl+C and it continues to hide the text typed. # So we reset the terminal via stty if the user does press Ctrl+C - trap '{ echo -e "\nNo password will be set" ; stty sane ; exit 1; }' INT + trap '{ echo -e "\nNot changed" ; stty sane ; exit 1; }' INT read -s -r -p "Enter New Password (Blank for no password): " PASSWORD echo "" @@ -186,689 +83,12 @@ SetWebPassword() { fi } -ProcessDNSSettings() { - source "${setupVars}" - - removeKey "${dnsmasqconfig}" "server" - - COUNTER=1 - while true ; do - var=PIHOLE_DNS_${COUNTER} - if [ -z "${!var}" ]; then - break; - fi - addKey "${dnsmasqconfig}" "server=${!var}" - (( COUNTER++ )) - done - - # The option LOCAL_DNS_PORT is deprecated - # We apply it once more, and then convert it into the current format - if [ -n "${LOCAL_DNS_PORT}" ]; then - addOrEditKeyValPair "${dnsmasqconfig}" "server" "127.0.0.1#${LOCAL_DNS_PORT}" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_${COUNTER}" "127.0.0.1#${LOCAL_DNS_PORT}" - removeKey "${setupVars}" "LOCAL_DNS_PORT" - fi - - removeKey "${dnsmasqconfig}" "domain-needed" - removeKey "${dnsmasqconfig}" "expand-hosts" - - if [[ "${DNS_FQDN_REQUIRED}" == true ]]; then - addKey "${dnsmasqconfig}" "domain-needed" - addKey "${dnsmasqconfig}" "expand-hosts" - fi - - removeKey "${dnsmasqconfig}" "bogus-priv" - - if [[ "${DNS_BOGUS_PRIV}" == true ]]; then - addKey "${dnsmasqconfig}" "bogus-priv" - fi - - removeKey "${dnsmasqconfig}" "dnssec" - removeKey "${dnsmasqconfig}" "trust-anchor" - - if [[ "${DNSSEC}" == true ]]; then - echo "dnssec -trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC683457104237C7F8EC8D -" >> "${dnsmasqconfig}" - fi - - removeKey "${dnsmasqconfig}" "host-record" - - if [ -n "${HOSTRECORD}" ]; then - addOrEditKeyValPair "${dnsmasqconfig}" "host-record" "${HOSTRECORD}" - fi - - # Setup interface listening behavior of dnsmasq - removeKey "${dnsmasqconfig}" "interface" - removeKey "${dnsmasqconfig}" "local-service" - removeKey "${dnsmasqconfig}" "except-interface" - removeKey "${dnsmasqconfig}" "bind-interfaces" - - if [[ "${DNSMASQ_LISTENING}" == "all" ]]; then - # Listen on all interfaces, permit all origins - addOrEditKeyValPair "${dnsmasqconfig}" "except-interface" "nonexisting" - elif [[ "${DNSMASQ_LISTENING}" == "local" ]]; then - # Listen only on all interfaces, but only local subnets - addKey "${dnsmasqconfig}" "local-service" - else - # Options "bind" and "single" - # Listen only on one interface - # Use eth0 as fallback interface if interface is missing in setupVars.conf - if [ -z "${PIHOLE_INTERFACE}" ]; then - PIHOLE_INTERFACE="eth0" - fi - - addOrEditKeyValPair "${dnsmasqconfig}" "interface" "${PIHOLE_INTERFACE}" - - if [[ "${DNSMASQ_LISTENING}" == "bind" ]]; then - # Really bind to interface - addKey "${dnsmasqconfig}" "bind-interfaces" - fi - fi - - if [[ "${CONDITIONAL_FORWARDING}" == true ]]; then - # Convert legacy "conditional forwarding" to rev-server configuration - # Remove any existing REV_SERVER settings - removeKey "${setupVars}" "REV_SERVER" - removeKey "${setupVars}" "REV_SERVER_DOMAIN" - removeKey "${setupVars}" "REV_SERVER_TARGET" - removeKey "${setupVars}" "REV_SERVER_CIDR" - - REV_SERVER=true - addOrEditKeyValPair "${setupVars}" "REV_SERVER" "true" - - REV_SERVER_DOMAIN="${CONDITIONAL_FORWARDING_DOMAIN}" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_DOMAIN" "${REV_SERVER_DOMAIN}" - - REV_SERVER_TARGET="${CONDITIONAL_FORWARDING_IP}" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_TARGET" "${REV_SERVER_TARGET}" - - #Convert CONDITIONAL_FORWARDING_REVERSE if necessary e.g: - # 1.1.168.192.in-addr.arpa to 192.168.1.1/32 - # 1.168.192.in-addr.arpa to 192.168.1.0/24 - # 168.192.in-addr.arpa to 192.168.0.0/16 - # 192.in-addr.arpa to 192.0.0.0/8 - if [[ "${CONDITIONAL_FORWARDING_REVERSE}" == *"in-addr.arpa" ]];then - arrRev=("${CONDITIONAL_FORWARDING_REVERSE//./ }") - case ${#arrRev[@]} in - 6 ) REV_SERVER_CIDR="${arrRev[3]}.${arrRev[2]}.${arrRev[1]}.${arrRev[0]}/32";; - 5 ) REV_SERVER_CIDR="${arrRev[2]}.${arrRev[1]}.${arrRev[0]}.0/24";; - 4 ) REV_SERVER_CIDR="${arrRev[1]}.${arrRev[0]}.0.0/16";; - 3 ) REV_SERVER_CIDR="${arrRev[0]}.0.0.0/8";; - esac - else - # Set REV_SERVER_CIDR to whatever value it was set to - REV_SERVER_CIDR="${CONDITIONAL_FORWARDING_REVERSE}" - fi - - # If REV_SERVER_CIDR is not converted by the above, then use the REV_SERVER_TARGET variable to derive it - if [ -z "${REV_SERVER_CIDR}" ]; then - # Convert existing input to /24 subnet (preserves legacy behavior) - # This sed converts "192.168.1.2" to "192.168.1.0/24" - # shellcheck disable=2001 - REV_SERVER_CIDR="$(sed "s+\\.[0-9]*$+\\.0/24+" <<< "${REV_SERVER_TARGET}")" - fi - addOrEditKeyValPair "${setupVars}" "REV_SERVER_CIDR" "${REV_SERVER_CIDR}" - - # Remove obsolete settings from setupVars.conf - removeKey "${setupVars}" "CONDITIONAL_FORWARDING" - removeKey "${setupVars}" "CONDITIONAL_FORWARDING_REVERSE" - removeKey "${setupVars}" "CONDITIONAL_FORWARDING_DOMAIN" - removeKey "${setupVars}" "CONDITIONAL_FORWARDING_IP" - fi - - removeKey "${dnsmasqconfig}" "rev-server" - - if [[ "${REV_SERVER}" == true ]]; then - addKey "${dnsmasqconfig}" "rev-server=${REV_SERVER_CIDR},${REV_SERVER_TARGET}" - if [ -n "${REV_SERVER_DOMAIN}" ]; then - # Forward local domain names to the CF target, too - addKey "${dnsmasqconfig}" "server=/${REV_SERVER_DOMAIN}/${REV_SERVER_TARGET}" - fi - - if [[ "${DNS_FQDN_REQUIRED}" != true ]]; then - # Forward unqualified names to the CF target only when the "never - # forward non-FQDN" option is unticked - addKey "${dnsmasqconfig}" "server=//${REV_SERVER_TARGET}" - fi - - fi - - # We need to process DHCP settings here as well to account for possible - # changes in the non-FQDN forwarding. This cannot be done in 01-pihole.conf - # as we don't want to delete all local=/.../ lines so it's much safer to - # simply rewrite the entire corresponding config file (which is what the - # DHCP settings subroutine is doing) - ProcessDHCPSettings -} - -SetDNSServers() { - # Save setting to file - removeKey "${setupVars}" "PIHOLE_DNS" - IFS=',' read -r -a array <<< "${args[2]}" - for index in "${!array[@]}" - do - # Replace possible "\#" by "#". This fixes AdminLTE#1427 - local ip - ip="${array[index]//\\#/#}" - - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - addOrEditKeyValPair "${setupVars}" "PIHOLE_DNS_$((index+1))" "${ip}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 - fi - done - - if [[ "${args[3]}" == "domain-needed" ]]; then - addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "true" - else - addOrEditKeyValPair "${setupVars}" "DNS_FQDN_REQUIRED" "false" - fi - - if [[ "${args[4]}" == "bogus-priv" ]]; then - addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "true" - else - addOrEditKeyValPair "${setupVars}" "DNS_BOGUS_PRIV" "false" - fi - - if [[ "${args[5]}" == "dnssec" ]]; then - addOrEditKeyValPair "${setupVars}" "DNSSEC" "true" - else - addOrEditKeyValPair "${setupVars}" "DNSSEC" "false" - fi - - if [[ "${args[6]}" == "rev-server" ]]; then - addOrEditKeyValPair "${setupVars}" "REV_SERVER" "true" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_CIDR" "${args[7]}" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_TARGET" "${args[8]}" - addOrEditKeyValPair "${setupVars}" "REV_SERVER_DOMAIN" "${args[9]}" - else - addOrEditKeyValPair "${setupVars}" "REV_SERVER" "false" - fi - - ProcessDNSSettings - - # Restart dnsmasq to load new configuration - RestartDNS -} - -SetExcludeDomains() { - addOrEditKeyValPair "${setupVars}" "API_EXCLUDE_DOMAINS" "${args[2]}" -} - -SetExcludeClients() { - addOrEditKeyValPair "${setupVars}" "API_EXCLUDE_CLIENTS" "${args[2]}" -} - -RestartDNS() { - "${PI_HOLE_BIN_DIR}"/pihole restartdns -} - -SetQueryLogOptions() { - addOrEditKeyValPair "${setupVars}" "API_QUERY_LOG_SHOW" "${args[2]}" -} - -ProcessDHCPSettings() { - source "${setupVars}" - - if [[ "${DHCP_ACTIVE}" == "true" ]]; then - interface="${PIHOLE_INTERFACE}" - - # Use eth0 as fallback interface - if [ -z ${interface} ]; then - interface="eth0" - fi - - if [[ "${PIHOLE_DOMAIN}" == "" ]]; then - PIHOLE_DOMAIN="lan" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DOMAIN" "${PIHOLE_DOMAIN}" - fi - - if [[ "${DHCP_LEASETIME}" == "0" ]]; then - leasetime="infinite" - elif [[ "${DHCP_LEASETIME}" == "" ]]; then - leasetime="24h" - addOrEditKeyValPair "${setupVars}" "DHCP_LEASETIME" "24" - else - leasetime="${DHCP_LEASETIME}h" - fi - - # Write settings to file - echo "############################################################################### -# DHCP SERVER CONFIG FILE AUTOMATICALLY POPULATED BY PI-HOLE WEB INTERFACE. # -# ANY CHANGES MADE TO THIS FILE WILL BE LOST ON CHANGE # -############################################################################### -dhcp-authoritative -dhcp-range=${DHCP_START},${DHCP_END},${leasetime} -dhcp-option=option:router,${DHCP_ROUTER} -dhcp-leasefile=/etc/pihole/dhcp.leases -#quiet-dhcp -" > "${dhcpconfig}" - chmod 644 "${dhcpconfig}" - - if [[ "${PIHOLE_DOMAIN}" != "none" ]]; then - echo "domain=${PIHOLE_DOMAIN}" >> "${dhcpconfig}" - - # When there is a Pi-hole domain set and "Never forward non-FQDNs" is - # ticked, we add `local=/domain/` to tell FTL that this domain is purely - # local and FTL may answer queries from /etc/hosts or DHCP but should - # never forward queries on that domain to any upstream servers - if [[ "${DNS_FQDN_REQUIRED}" == true ]]; then - echo "local=/${PIHOLE_DOMAIN}/" >> "${dhcpconfig}" - fi - fi - - # Sourced from setupVars - # shellcheck disable=SC2154 - if [[ "${DHCP_rapid_commit}" == "true" ]]; then - echo "dhcp-rapid-commit" >> "${dhcpconfig}" - fi - - if [[ "${DHCP_IPv6}" == "true" ]]; then - echo "#quiet-dhcp6 -#enable-ra -dhcp-option=option6:dns-server,[::] -dhcp-range=::,constructor:${interface},ra-names,ra-stateless,64 - -" >> "${dhcpconfig}" - fi - - else - if [[ -f "${dhcpconfig}" ]]; then - rm "${dhcpconfig}" &> /dev/null - fi - fi -} - -EnableDHCP() { - addOrEditKeyValPair "${setupVars}" "DHCP_ACTIVE" "true" - addOrEditKeyValPair "${setupVars}" "DHCP_START" "${args[2]}" - addOrEditKeyValPair "${setupVars}" "DHCP_END" "${args[3]}" - addOrEditKeyValPair "${setupVars}" "DHCP_ROUTER" "${args[4]}" - addOrEditKeyValPair "${setupVars}" "DHCP_LEASETIME" "${args[5]}" - addOrEditKeyValPair "${setupVars}" "PIHOLE_DOMAIN" "${args[6]}" - addOrEditKeyValPair "${setupVars}" "DHCP_IPv6" "${args[7]}" - addOrEditKeyValPair "${setupVars}" "DHCP_rapid_commit" "${args[8]}" - - # Remove possible old setting from file - removeKey "${dnsmasqconfig}" "dhcp-" - removeKey "${dnsmasqconfig}" "quiet-dhcp" - - # If a DHCP client claims that its name is "wpad", ignore that. - # This fixes a security hole. see CERT Vulnerability VU#598349 - # We also ignore "localhost" as Windows behaves strangely if a - # device claims this host name - addKey "${dnsmasqconfig}" "dhcp-name-match=set:hostname-ignore,wpad -dhcp-name-match=set:hostname-ignore,localhost -dhcp-ignore-names=tag:hostname-ignore" - - ProcessDHCPSettings - - RestartDNS -} - -DisableDHCP() { - addOrEditKeyValPair "${setupVars}" "DHCP_ACTIVE" "false" - - # Remove possible old setting from file - removeKey "${dnsmasqconfig}" "dhcp-" - removeKey "${dnsmasqconfig}" "quiet-dhcp" - - ProcessDHCPSettings - - RestartDNS -} - -SetWebUILayout() { - addOrEditKeyValPair "${setupVars}" "WEBUIBOXEDLAYOUT" "${args[2]}" -} - -SetWebUITheme() { - addOrEditKeyValPair "${setupVars}" "WEBTHEME" "${args[2]}" -} - -CheckUrl(){ - local regex check_url - # Check for characters NOT allowed in URLs - regex="[^a-zA-Z0-9:/?&%=~._()-;]" - - # this will remove first @ that is after schema and before domain - # \1 is optional schema, \2 is userinfo - check_url="$( sed -re 's#([^:/]*://)?([^/]+)@#\1\2#' <<< "$1" )" - - if [[ "${check_url}" =~ ${regex} ]]; then - return 1 - else - return 0 - fi -} - -CustomizeAdLists() { - local address - address="${args[3]}" - local comment - comment="${args[4]}" - - if CheckUrl "${address}"; then - if [[ "${args[2]}" == "enable" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'" - elif [[ "${args[2]}" == "disable" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'" - elif [[ "${args[2]}" == "add" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')" - elif [[ "${args[2]}" == "del" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'" - else - echo "Not permitted" - return 1 - fi - else - echo "Invalid Url" - return 1 - fi -} - -AddDHCPStaticAddress() { - mac="${args[2]}" - ip="${args[3]}" - host="${args[4]}" - - if [[ "${ip}" == "noip" ]]; then - # Static host name - echo "dhcp-host=${mac},${host}" >> "${dhcpstaticconfig}" - elif [[ "${host}" == "nohost" ]]; then - # Static IP - echo "dhcp-host=${mac},${ip}" >> "${dhcpstaticconfig}" - else - # Full info given - echo "dhcp-host=${mac},${ip},${host}" >> "${dhcpstaticconfig}" - fi -} - -RemoveDHCPStaticAddress() { - mac="${args[2]}" - if [[ "$mac" =~ ^([0-9A-Fa-f]{2}[:-]){5}([0-9A-Fa-f]{2})$ ]]; then - sed -i "/dhcp-host=${mac}.*/d" "${dhcpstaticconfig}" - else - echo " ${CROSS} Invalid Mac Passed!" - exit 1 - fi - -} - -SetListeningMode() { - source "${setupVars}" - - if [[ "$3" == "-h" ]] || [[ "$3" == "--help" ]]; then - echo "Usage: pihole -a -i [interface] -Example: 'pihole -a -i local' -Specify dnsmasq's network interface listening behavior - -Interfaces: - local Only respond to queries from devices that - are at most one hop away (local devices) - single Respond only on interface ${PIHOLE_INTERFACE} - bind Bind only on interface ${PIHOLE_INTERFACE} - all Listen on all interfaces, permit all origins" - exit 0 - fi - - if [[ "${args[2]}" == "all" ]]; then - echo -e " ${INFO} Listening on all interfaces, permitting all origins. Please use a firewall!" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "all" - elif [[ "${args[2]}" == "local" ]]; then - echo -e " ${INFO} Listening on all interfaces, permitting origins from one hop away (LAN)" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "local" - elif [[ "${args[2]}" == "bind" ]]; then - echo -e " ${INFO} Binding on interface ${PIHOLE_INTERFACE}" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "bind" - else - echo -e " ${INFO} Listening only on interface ${PIHOLE_INTERFACE}" - addOrEditKeyValPair "${setupVars}" "DNSMASQ_LISTENING" "single" - fi - - # Don't restart DNS server yet because other settings - # will be applied afterwards if "-web" is set - if [[ "${args[3]}" != "-web" ]]; then - ProcessDNSSettings - # Restart dnsmasq to load new configuration - RestartDNS - fi -} - -Teleporter() { - local filename - filename="${args[2]}" - if [[ -z "${filename}" ]]; then - local datetimestamp - local host - datetimestamp=$(date "+%Y-%m-%d_%H-%M-%S") - host=$(hostname) - host="${host//./_}" - filename="pi-hole-${host:-noname}-teleporter_${datetimestamp}.tar.gz" - fi - php "${webroot}/admin/scripts/pi-hole/php/teleporter.php" > "${filename}" -} - -checkDomain() -{ - local domain validDomain - # Convert to lowercase - domain="${1,,}" - validDomain=$(grep -P "^((-|_)*[a-z0-9]((-|_)*[a-z0-9)*(-|_)*)(\\.(-|_)*([a-z0-9]((-|_)*[a-z0-9])*))*$" <<< "${domain}") # Valid chars check - validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label - echo "${validDomain}" -} - -escapeDots() -{ - # SC suggest bashism ${variable//search/replace} - # shellcheck disable=SC2001 - escaped=$(echo "$1" | sed 's/\./\\./g') - echo "${escaped}" -} - -addAudit() -{ - shift # skip "-a" - shift # skip "audit" - local domains validDomain - domains="" - for domain in "$@" - do - # Check domain to be added. Only continue if it is valid - validDomain="$(checkDomain "${domain}")" - if [[ -n "${validDomain}" ]]; then - # Put comma in between domains when there is - # more than one domains to be added - # SQL INSERT allows adding multiple rows at once using the format - ## INSERT INTO table (domain) VALUES ('abc.de'),('fgh.ij'),('klm.no'),('pqr.st'); - if [[ -n "${domains}" ]]; then - domains="${domains}," - fi - domains="${domains}('${domain}')" - fi - done - # Insert only the domain here. The date_added field will be - # filled with its default value (date_added = current timestamp) - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};" -} - -clearAudit() -{ - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;" -} - -SetPrivacyLevel() { - # Set privacy level. Minimum is 0, maximum is 3 - if [ "${args[2]}" -ge 0 ] && [ "${args[2]}" -le 3 ]; then - addOrEditKeyValPair "${FTLconf}" "PRIVACYLEVEL" "${args[2]}" - pihole restartdns reload-lists - fi -} - -AddCustomDNSAddress() { - echo -e " ${TICK} Adding custom DNS entry..." - - ip="${args[2]}" - host="${args[3]}" - reload="${args[4]}" - - validHost="$(checkDomain "${host}")" - if [[ -n "${validHost}" ]]; then - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - echo "${ip} ${validHost}" >> "${dnscustomfile}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi - - # Restart dnsmasq to load new custom DNS entries only if $reload not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - -RemoveCustomDNSAddress() { - echo -e " ${TICK} Removing custom DNS entry..." - - ip="${args[2]}" - host="${args[3]}" - reload="${args[4]}" - - validHost="$(checkDomain "${host}")" - if [[ -n "${validHost}" ]]; then - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - validHost=$(escapeDots "${validHost}") - sed -i "/^${ip} ${validHost}$/Id" "${dnscustomfile}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi - - # Restart dnsmasq to load new custom DNS entries only if reload is not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - -AddCustomCNAMERecord() { - echo -e " ${TICK} Adding custom CNAME record..." - - domain="${args[2]}" - target="${args[3]}" - reload="${args[4]}" - - validDomain="$(checkDomain "${domain}")" - if [[ -n "${validDomain}" ]]; then - validTarget="$(checkDomain "${target}")" - if [[ -n "${validTarget}" ]]; then - if [ "${validDomain}" = "${validTarget}" ]; then - echo " ${CROSS} Domain and target are the same. This would cause a DNS loop." - exit 1 - else - echo "cname=${validDomain},${validTarget}" >> "${dnscustomcnamefile}" - fi - else - echo " ${CROSS} Invalid Target Passed!" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi - # Restart dnsmasq to load new custom CNAME records only if reload is not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - -RemoveCustomCNAMERecord() { - echo -e " ${TICK} Removing custom CNAME record..." - - domain="${args[2]}" - target="${args[3]}" - reload="${args[4]}" - - validDomain="$(checkDomain "${domain}")" - if [[ -n "${validDomain}" ]]; then - validTarget="$(checkDomain "${target}")" - if [[ -n "${validTarget}" ]]; then - validDomain=$(escapeDots "${validDomain}") - validTarget=$(escapeDots "${validTarget}") - sed -i "/^cname=${validDomain},${validTarget}$/Id" "${dnscustomcnamefile}" - else - echo " ${CROSS} Invalid Target Passed!" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi - - # Restart dnsmasq to update removed custom CNAME records only if $reload not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - -SetRateLimit() { - local rate_limit_count rate_limit_interval reload - rate_limit_count="${args[2]}" - rate_limit_interval="${args[3]}" - reload="${args[4]}" - - # Set rate-limit setting inf valid - if [ "${rate_limit_count}" -ge 0 ] && [ "${rate_limit_interval}" -ge 0 ]; then - addOrEditKeyValPair "${FTLconf}" "RATE_LIMIT" "${rate_limit_count}/${rate_limit_interval}" - fi - - # Restart FTL to update rate-limit settings only if $reload not false - if [[ ! $reload == "false" ]]; then - RestartDNS - fi -} - main() { args=("$@") case "${args[1]}" in "-p" | "password" ) SetWebPassword;; - "-c" | "celsius" ) unit="C"; SetTemperatureUnit;; - "-f" | "fahrenheit" ) unit="F"; SetTemperatureUnit;; - "-k" | "kelvin" ) unit="K"; SetTemperatureUnit;; - "setdns" ) SetDNSServers;; - "setexcludedomains" ) SetExcludeDomains;; - "setexcludeclients" ) SetExcludeClients;; - "restartdns" ) RestartDNS;; - "setquerylog" ) SetQueryLogOptions;; - "enabledhcp" ) EnableDHCP;; - "disabledhcp" ) DisableDHCP;; - "layout" ) SetWebUILayout;; - "theme" ) SetWebUITheme;; "-h" | "--help" ) helpFunc;; - "addstaticdhcp" ) AddDHCPStaticAddress;; - "removestaticdhcp" ) RemoveDHCPStaticAddress;; - "-i" | "interface" ) SetListeningMode "$@";; - "-t" | "teleporter" ) Teleporter;; - "adlist" ) CustomizeAdLists;; - "audit" ) addAudit "$@";; - "clearaudit" ) clearAudit;; - "-l" | "privacylevel" ) SetPrivacyLevel;; - "addcustomdns" ) AddCustomDNSAddress;; - "removecustomdns" ) RemoveCustomDNSAddress;; - "addcustomcname" ) AddCustomCNAMERecord;; - "removecustomcname" ) RemoveCustomCNAMERecord;; - "ratelimit" ) SetRateLimit;; * ) helpFunc;; esac From 11679a5188c1e29fcaa13fa5c29204443349a794 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 28 May 2023 23:31:51 +0100 Subject: [PATCH 095/462] @MrDuck2742 dropped a `]` on the floor without me seeing. I picked it up and put it back again Signed-off-by: Adam Warner --- advanced/Scripts/webpage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 7680590f..c92d0458 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -664,7 +664,7 @@ checkDomain() local domain validDomain # Convert to lowercase domain="${1,,}" - validDomain=$(grep -P "^((-|_)*[a-z0-9]((-|_)*[a-z0-9)*(-|_)*)(\\.(-|_)*([a-z0-9]((-|_)*[a-z0-9])*))*$" <<< "${domain}") # Valid chars check + validDomain=$(grep -P "^((-|_)*[a-z0-9]((-|_)*[a-z0-9])*(-|_)*)(\\.(-|_)*([a-z0-9]((-|_)*[a-z0-9])*))*$" <<< "${domain}") # Valid chars check validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label echo "${validDomain}" } From 494734bf27f836d2e5f17df1567d79fc37ad9de0 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 28 May 2023 23:40:47 +0100 Subject: [PATCH 096/462] Add in a test case Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index c1288287..36a4f0d2 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -1151,3 +1151,30 @@ def test_package_manager_has_web_deps(host): assert "No package" not in output.stdout assert output.rc == 0 + + +def test_webpage_sh_valid_domain(host): + """Confirms checkDomain function in webpage.sh works as expected""" + check1 = host.run( + """ + source /opt/pihole/webpage.sh + checkDomain "pi-hole.net" + """ + ) + check2 = host.run( + """ + source /opt/pihole/webpage.sh + checkDomain "ab.pi-hole.net" + """ + ) + + check3 = host.run( + """ + source /opt/pihole/webpage.sh + checkDomain "abc.pi-hole.net" + """ + ) + + assert "pi-hole.net" in check1.stdout + assert "ab.pi-hole.net" in check2.stdout + assert "abc.pi-hole.net" in check3.stdout From 6c302c9bc811a944cc721f640e3f5429db25869e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 30 May 2023 17:01:58 +0200 Subject: [PATCH 097/462] Adlist properties need to be stored in the new ("temp") database rather than the old. Signed-off-by: DL6ER --- gravity.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index fe2097dd..2bd21d01 100755 --- a/gravity.sh +++ b/gravity.sh @@ -229,7 +229,7 @@ database_table_from_file() { # Check if a column with name ${2} exists in gravity table with name ${1} gravity_column_exists() { - output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) if [[ "${output}" == "1" ]]; then return 0 # Bash 0 is success fi @@ -244,11 +244,11 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to update number of domains in adlist with ID ${1} in database ${gravityDBfile}\\n ${output}" + echo -e "\\n ${CROSS} Unable to update number of domains in adlist with ID ${1} in database ${gravityTEMPfile}\\n ${output}" gravity_Cleanup "error" fi } @@ -260,11 +260,11 @@ database_adlist_status() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to update status of adlist with ID ${1} in database ${gravityDBfile}\\n ${output}" + echo -e "\\n ${CROSS} Unable to update status of adlist with ID ${1} in database ${gravityTEMPfile}\\n ${output}" gravity_Cleanup "error" fi } From f7b9d70054758137d60318e6df32ebb0d92987fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 30 May 2023 21:39:10 +0200 Subject: [PATCH 098/462] Remove man page for pihole-FTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 8 +- manpages/pihole-FTL.8 | 154 ----------------------------- test/test_any_automated_install.py | 5 - 3 files changed, 5 insertions(+), 162 deletions(-) delete mode 100644 manpages/pihole-FTL.8 diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 984fe0c0..28813f1e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1253,12 +1253,14 @@ install_manpage() { fi # Testing complete, copy the files & update the man db install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 - # remove previously installed "pihole-FTL.conf.5" man page + # remove previously installed man pages if [[ -f "/usr/local/share/man/man5/pihole-FTL.conf.5" ]]; then rm /usr/local/share/man/man5/pihole-FTL.conf.5 fi + if [[ -f "/usr/local/share/man/man5/pihole-FTL.8" ]]; then + rm /usr/local/share/man/man5/pihole-FTL.8 + fi if mandb -q &>/dev/null; then # Updated successfully @@ -1267,7 +1269,7 @@ install_manpage() { else # Something is wrong with the system's man installation, clean up # our files, (leave everything how we found it). - rm /usr/local/share/man/man8/pihole.8 /usr/local/share/man/man8/pihole-FTL.8 + rm /usr/local/share/man/man8/pihole.8 printf "%b %b man page db not updated, man pages not installed\\n" "${OVER}" "${CROSS}" fi } diff --git a/manpages/pihole-FTL.8 b/manpages/pihole-FTL.8 deleted file mode 100644 index c1b7550f..00000000 --- a/manpages/pihole-FTL.8 +++ /dev/null @@ -1,154 +0,0 @@ -.TH "Pihole-FTL" "8" "pihole-FTL" "Pi-hole" "November 2020" -.SH "NAME" -pihole-FTL - Pi-hole : The Faster-Than-Light (FTL) Engine -.br -.SH "SYNOPSIS" -\fBservice pihole-FTL \fR(\fBstart\fR|\fBstop\fR|\fBrestart\fR) -.br - -\fBpihole-FTL debug\fR -.br -\fBpihole-FTL test\fR -.br -\fBpihole-FTL -v|-vv\fR -.br -\fBpihole-FTL -t\fR -.br -\fBpihole-FTL -b\fR -.br -\fBpihole-FTL -f\fR -.br -\fBpihole-FTL -h\fR -.br -\fBpihole-FTL dnsmasq-test\fR -.br -\fBpihole-FTL regex-test str\fR -.br -\fBpihole-FTL regex-test str rgx\fR -.br -\fBpihole-FTL lua\fR -.br -\fBpihole-FTL luac\fR -.br -\fBpihole-FTL dhcp-discover\fR -.br -\fBpihole-FTL --\fR (\fBoptions\fR) -.br - -.SH "DESCRIPTION" -Pi-hole : The Faster-Than-Light (FTL) Engine is a lightweight, purpose-built daemon used to provide statistics needed for the Pi-hole Web Interface, and its API can be easily integrated into your own projects. Although it is an optional component of the Pi-hole ecosystem, it will be installed by default to provide statistics. As the name implies, FTL does its work \fIvery\fR \fIquickly\fR! -.br - -Usage -.br - -\fBservice pihole-FTL start\fR -.br - Start the pihole-FTL daemon -.br - -\fBservice pihole-FTL stop\fR -.br - Stop the pihole-FTL daemon -.br - -\fBservice pihole-FTL restart\fR -.br - If the pihole-FTP daemon is running, stop and then start, otherwise start. -.br - -Command line arguments -.br - -\fBdebug\fR -.br - Don't go into daemon mode (stay in foreground) + more verbose logging -.br - -\fBtest\fR -.br - Start FTL and process everything, but shut down immediately afterwards -.br - -\fB-v, version\fR -.br - Don't start FTL, show only version -.br - -\fB-vv\fR -.br - Don't start FTL, show verbose version information of embedded applications -.br - -\fB-t, tag\fR -.br - Don't start FTL, show only git tag -.br - -\fB-b, branch\fR -.br - Don't start FTL, show only git branch FTL was compiled from -.br - -\fB-f, no-daemon\fR -.br - Don't go into background (daemon mode) -.br - -\fB-h, help\fR -.br - Don't start FTL, show help -.br - -\fBdnsmasq-test\fR -.br - Test resolver config file syntax -.br - -\fBregex-test str\fR -.br - Test str against all regular expressions in the database -.br - -\fBregex-test str rgx\fR -.br - Test str against regular expression given by rgx -.br - -\fBlua\fR -.br - Start the embedded Lua interpreter -.br - -\fBluac\fR -.br - Execute the embedded Lua compiler -.br - -\fBdhcp-discover\fR -.br - Discover DHCP servers in the local network -.br - -\fB--\fR (options) -.br - Pass options to internal dnsmasq resolver -.br -.SH "EXAMPLE" -Command line arguments can be arbitrarily combined, e.g: -.br - -\fBpihole-FTL debug test\fR -.br - -Start ftl in foreground with more verbose logging, process everything and shutdown immediately -.br -.SH "SEE ALSO" -\fBpihole\fR(8) -.br -\fBFor FTL's config options please see https://docs.pi-hole.net/ftldns/configfile/\fR -.br -.SH "COLOPHON" - -Get sucked into the latest news and community activity by entering Pi-hole's orbit. Information about Pi-hole, and the latest version of the software can be found at https://pi-hole.net -.br diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 45cb01c5..3626ce6f 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -174,11 +174,6 @@ def test_installPihole_fresh_install_readableFiles(host): ) actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc - check_man = test_cmd.format( - "r", "/usr/local/share/man/man8/pihole-FTL.8", piholeuser - ) - actual_rc = host.run(check_man).rc - assert exit_status_success == actual_rc # check not readable sudoers file check_sudo = test_cmd.format("r", "/etc/sudoers.d/pihole", piholeuser) actual_rc = host.run(check_sudo).rc From b2c73f8325ae059a4dec86fca42c7dead6839b31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 29 May 2023 22:02:38 +0200 Subject: [PATCH 099/462] Do not try to remove stale lables on PRs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index fe28112c..af3177d4 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -29,8 +29,11 @@ jobs: operations-per-run: 300 close-issue-reason: 'not_planned' - remove_stale: # trigger "stale" removal immediately when stale issues are commented on - if: github.event_name == 'issue_comment' + remove_stale: + # trigger "stale" removal immediately when stale issues are commented on + # we need to explicitly check that the trigger does not run on comment on a PR as + # 'issue_comment' triggers on issues AND PR comments + if: github.event_name == 'issue_comment' && ${{ !github.event.issue.pull_request }} permissions: contents: read # for actions/checkout issues: write # to edit issues label From 364537b32459d3c5871f8dde0d9cad5b7e12b008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 31 May 2023 22:14:05 +0200 Subject: [PATCH 100/462] Use env variable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index af3177d4..0d06ee22 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -23,7 +23,7 @@ jobs: days-before-stale: 30 days-before-close: 5 stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Please comment or update this issue or it will be closed in 5 days.' - stale-issue-label: $stale_label + stale-issue-label: '${{ env.stale_label }}' exempt-issue-labels: 'Internal, Fixed in next release, Bug: Confirmed, Documentation Needed' exempt-all-issue-assignees: true operations-per-run: 300 @@ -42,7 +42,6 @@ jobs: - name: Checkout uses: actions/checkout@v3.5.2 - name: Remove 'stale' label - run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label - env: + run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 792b0d419b20875dab023e311d532318f80ab2ff Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 1 Jun 2023 19:29:47 +0100 Subject: [PATCH 101/462] Test fixes Signed-off-by: Adam Warner --- automated install/basic-install.sh | 13 ------------- test/test_any_automated_install.py | 27 --------------------------- test/test_any_utils.py | 2 +- 3 files changed, 1 insertion(+), 41 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a6f6b865..6663bbad 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1566,17 +1566,6 @@ create_pihole_user() { fi } -finalExports() { - # shellcheck source=advanced/Scripts/webpage.sh - source "${PI_HOLE_LOCAL_REPO}/advanced/Scripts/webpage.sh" - - # Look for DNS server settings which would have to be reapplied - ProcessDNSSettings - - # Look for DHCP server settings which would have to be reapplied - ProcessDHCPSettings -} - # Install the logrotate script installLogrotate() { local str="Installing latest logrotate script" @@ -1649,8 +1638,6 @@ installPihole() { # install a man page entry for pihole install_manpage - - finalExports } # SELinux diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 07a18100..3626ce6f 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -781,30 +781,3 @@ def test_package_manager_has_web_deps(host): assert "No package" not in output.stdout assert output.rc == 0 - - -def test_webpage_sh_valid_domain(host): - """Confirms checkDomain function in webpage.sh works as expected""" - check1 = host.run( - """ - source /opt/pihole/webpage.sh - checkDomain "pi-hole.net" - """ - ) - check2 = host.run( - """ - source /opt/pihole/webpage.sh - checkDomain "ab.pi-hole.net" - """ - ) - - check3 = host.run( - """ - source /opt/pihole/webpage.sh - checkDomain "abc.pi-hole.net" - """ - ) - - assert "pi-hole.net" in check1.stdout - assert "ab.pi-hole.net" in check2.stdout - assert "abc.pi-hole.net" in check3.stdout diff --git a/test/test_any_utils.py b/test/test_any_utils.py index efab3760..1efd77c3 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -151,4 +151,4 @@ def test_getFTLConfigValue_getFTLConfigValue(host): """ ) - assert '[ "9.9.9.9" ]' in output.stdout + assert '[ 9.9.9.9 ]' in output.stdout From 001024b4da4be7de189c0ae3b7dde2cbe6aa1ca3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 1 Jun 2023 20:41:54 +0200 Subject: [PATCH 102/462] Fix Tests No2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 1efd77c3..154b5e16 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -151,4 +151,4 @@ def test_getFTLConfigValue_getFTLConfigValue(host): """ ) - assert '[ 9.9.9.9 ]' in output.stdout + assert "[ 9.9.9.9 ]" in output.stdout From a4322c624f5047123c5b8806f7e9222bfa89568f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 1 Jun 2023 22:00:40 +0200 Subject: [PATCH 103/462] Fix gravity in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 3626ce6f..c1ef8af3 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -79,6 +79,11 @@ def test_installPihole_fresh_install_readableFiles(host): host.run("command -v apt-get > /dev/null && apt-get install -qq man") host.run("command -v dnf > /dev/null && dnf install -y man") host.run("command -v yum > /dev/null && yum install -y man") + # Workaround to get FTLv6 installed until it reaches master branch + host.run(""" + echo "new/http" > /etc/pihole/ftlbranch + """ + ) install = host.run( """ export TERM=xterm From 0becc7615a74e50630a153030e4d0fa7aba87e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 1 Jun 2023 23:51:03 +0200 Subject: [PATCH 104/462] Run prestart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 7 ++++--- test/test_any_utils.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index c1ef8af3..df9c4721 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -64,7 +64,7 @@ def test_installPihole_fresh_install_readableFiles(host): mock_command("dialog", {"*": ("", "0")}, host) # mock git pull mock_command_passthrough("git", {"pull": ("", "0")}, host) - # mock systemctl to not start lighttpd and FTL + # mock systemctl to not start FTL mock_command_2( "systemctl", { @@ -95,6 +95,7 @@ def test_installPihole_fresh_install_readableFiles(host): runUnattended=true useUpdateVars=true main + /opt/pihole/pihole-FTL-prestart.sh """ ) assert 0 == install.rc @@ -507,9 +508,9 @@ def test_FTL_download_aarch64_no_errors(host): assert "error" not in download_binary.stdout.lower() -def test_FTL_binary_installed_and_responsive_no_errors(host): +def test_FTL_development_binary_installed_and_responsive_no_errors(host): """ - confirms FTL binary is copied and functional in installed location + confirms FTL development binary is copied and functional in installed location """ host.run( """ diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 154b5e16..8c1ea521 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -129,7 +129,7 @@ def test_getFTLPIDFile_and_getFTLPID_custom(host): def test_getFTLConfigValue_getFTLConfigValue(host): """ Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) - Requires FTL to be installed, so we do that first (taken from test_FTL_binary_installed_and_responsive_no_errors) + Requires FTL to be installed, so we do that first (taken from test_FTL_development_binary_installed_and_responsive_no_errors) """ host.run( """ From 7da57c6acddf6d77610b934f7902e134ee9ef377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 2 Jun 2023 00:03:22 +0200 Subject: [PATCH 105/462] Don't check and install old FTL config file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 10 ---------- test/test_any_automated_install.py | 12 +++--------- test/test_any_utils.py | 3 ++- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 6663bbad..37e97e90 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -78,7 +78,6 @@ PI_HOLE_FILES=(chronometer list piholeDebug piholeLogFlush setupLCD update versi PI_HOLE_INSTALL_DIR="/opt/pihole" PI_HOLE_CONFIG_DIR="/etc/pihole" PI_HOLE_BIN_DIR="/usr/local/bin" -FTL_CONFIG_FILE="${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" if [ -z "$useUpdateVars" ]; then useUpdateVars=false fi @@ -1192,15 +1191,6 @@ installConfigs() { echo "${DNS_SERVERS}" > "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" - # Install template file if it does not exist - if [[ ! -r "${FTL_CONFIG_FILE}" ]]; then - install -d -m 0755 ${PI_HOLE_CONFIG_DIR} - if ! install -T -o pihole -m 664 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL.conf" "${FTL_CONFIG_FILE}" &>/dev/null; then - printf " %b Error: Unable to initialize configuration file %s/pihole-FTL.conf\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}" - return 1 - fi - fi - # Install empty custom.list file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then if ! install -o root -m 644 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index df9c4721..d5f76ba3 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -80,9 +80,10 @@ def test_installPihole_fresh_install_readableFiles(host): host.run("command -v dnf > /dev/null && dnf install -y man") host.run("command -v yum > /dev/null && yum install -y man") # Workaround to get FTLv6 installed until it reaches master branch - host.run(""" - echo "new/http" > /etc/pihole/ftlbranch + host.run( """ + echo "new/http" > /etc/pihole/ftlbranch + """ ) install = host.run( """ @@ -141,13 +142,6 @@ def test_installPihole_fresh_install_readableFiles(host): check_macvendor = test_cmd.format("r", "/etc/pihole/macvendor.db", piholeuser) actual_rc = host.run(check_macvendor).rc assert exit_status_success == actual_rc - # readable and writeable pihole-FTL.conf - check_FTLconf = test_cmd.format("r", "/etc/pihole/pihole-FTL.conf", piholeuser) - actual_rc = host.run(check_FTLconf).rc - assert exit_status_success == actual_rc - check_FTLconf = test_cmd.format("w", "/etc/pihole/pihole-FTL.conf", piholeuser) - actual_rc = host.run(check_FTLconf).rc - assert exit_status_success == actual_rc # check readable and executable /etc/init.d/pihole-FTL check_init = test_cmd.format("x", "/etc/init.d/pihole-FTL", piholeuser) actual_rc = host.run(check_init).rc diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 8c1ea521..0f300457 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -129,7 +129,8 @@ def test_getFTLPIDFile_and_getFTLPID_custom(host): def test_getFTLConfigValue_getFTLConfigValue(host): """ Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) - Requires FTL to be installed, so we do that first (taken from test_FTL_development_binary_installed_and_responsive_no_errors) + Requires FTL to be installed, so we do that first + (taken from test_FTL_development_binary_installed_and_responsive_no_errors) """ host.run( """ From ec82aec55fab2b0e751b3ff181d8aab8b653feab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 2 Jun 2023 00:13:57 +0200 Subject: [PATCH 106/462] centos_common_support.py was removed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/tox.fedora_38.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tox.fedora_38.ini b/test/tox.fedora_38.ini index 0aa7612e..d596092c 100644 --- a/test/tox.fedora_38.ini +++ b/test/tox.fedora_38.ini @@ -5,4 +5,4 @@ envlist = py3 allowlist_externals = docker deps = -rrequirements.txt commands = docker buildx build --load --progress plain -f _fedora_38.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From b0fa3795e9712d333b0d39db0caec1e2b75267c4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 3 Jun 2023 10:58:23 +0000 Subject: [PATCH 107/462] Bump tox from 4.5.1 to 4.5.2 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.5.1 to 4.5.2. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.5.1...4.5.2) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index f11d1253..de54d22c 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.3.1 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.5.1 +tox == 4.5.2 From 667418c71d0b29c927eb0c16eea4c4dadfd974a8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 31 May 2023 02:22:53 +0200 Subject: [PATCH 108/462] Use new FTL binary names Signed-off-by: DL6ER --- automated install/basic-install.sh | 46 ++++++++++++++---------------- test/test_any_automated_install.py | 24 +++++----------- 2 files changed, 29 insertions(+), 41 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 37e97e90..681b6798 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1951,39 +1951,29 @@ get_binary_name() { if [[ "${lib}" == "/lib/ld-linux-aarch64.so.1" ]]; then printf "%b %b Detected AArch64 (64 Bit ARM) processor\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-aarch64-linux-gnu" + l_binary="pihole-FTL-arm64" elif [[ "${lib}" == "/lib/ld-linux-armhf.so.3" ]]; then # Hard-float available: Use gnueabihf binaries # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) if [[ "${rev}" -gt 7 ]]; then printf "%b %b Detected ARMv8 (or newer) processor\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-armv8-linux-gnueabihf" + l_binary="pihole-FTL-armv8" elif [[ "${rev}" -eq 7 ]]; then # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) printf "%b %b Detected ARMv7 processor (with hard-float support)\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-armv7-linux-gnueabihf" + l_binary="pihole-FTL-armv7" else # Otherwise, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) printf "%b %b Detected ARMv6 processor (with hard-float support)\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-armv6-linux-gnueabihf" + l_binary="pihole-FTL-armv6" fi else - # No hard-float support found: Use gnueabi binaries - # Use the ARMv4-compliant binary only if we detected an ARMv4T core - if [[ "${rev}" -eq 4 ]]; then - printf "%b %b Detected ARMv4 processor\\n" "${OVER}" "${TICK}" - # set the binary to be used - l_binary="pihole-FTL-armv4-linux-gnueabi" - # Otherwise, use the ARMv5 binary. To date (end of 2020), all modern ARM processors - # are backwards-compatible to the ARMv5 - else - printf "%b %b Detected ARMv5 (or newer) processor\\n" "${OVER}" "${TICK}" - # set the binary to be used - l_binary="pihole-FTL-armv5-linux-gnueabi" - fi + # No hard-float support found + printf "%b %b%b ARM processor without hard-float support detected%b\\n" "${OVER}" "${COL_LIGHT_RED}" "${CROSS}" "${COL_NC}" + l_binary="" fi elif [[ "${machine}" == "x86_64" ]]; then # This gives the processor of packages dpkg installs (for example, "i386") @@ -1996,16 +1986,16 @@ get_binary_name() { # in the past (see https://github.com/pi-hole/pi-hole/pull/2004) if [[ "${dpkgarch}" == "i386" ]]; then printf "%b %b Detected 32bit (i686) processor\\n" "${OVER}" "${TICK}" - l_binary="pihole-FTL-linux-x86_32" + l_binary="pihole-FTL-386" else # 64bit printf "%b %b Detected x86_64 processor\\n" "${OVER}" "${TICK}" # set the binary to be used - l_binary="pihole-FTL-linux-x86_64" + l_binary="pihole-FTL-amd64" fi elif [[ "${machine}" == "riscv64" ]]; then printf "%b %b Detected riscv64 processor\\n" "${OVER}" "${TICK}" - l_binary="pihole-FTL-riscv64-linux-gnu" + l_binary="pihole-FTL-riscv64" else # Something else - we try to use 32bit executable and warn the user if [[ ! "${machine}" == "i686" ]]; then @@ -2015,7 +2005,7 @@ get_binary_name() { else printf "%b %b Detected 32bit (i686) processor\\n" "${OVER}" "${TICK}" fi - l_binary="pihole-FTL-linux-x86_32" + l_binary="pihole-FTL-linux-386" fi # Returning a string value via echo @@ -2180,6 +2170,16 @@ main() { fi fi + # Check if there is a usable FTL binary available on this architecture - do + # this early on as FTL is a hard dependency for Pi-hole + local funcOutput + funcOutput=$(get_binary_name) #Store output of get_binary_name here + # Abort early if this processor is not supported (get_binary_name returnS empty string) + if [[ "${funcOutput}" == "" ]]; then + printf " %b Upgrade/install aborted\\n" "${CROSS}" "${DISTRO_NAME}" + exit 1 + fi + # Check if SELinux is Enforcing and exit before doing anything else checkSelinux @@ -2249,9 +2249,7 @@ main() { # Create the pihole user create_pihole_user - # Check if FTL is installed - do this early on as FTL is a hard dependency for Pi-hole - local funcOutput - funcOutput=$(get_binary_name) #Store output of get_binary_name here + # Download and install FTL local binary binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL) local theRest diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index d5f76ba3..4a86d28e 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -268,9 +268,9 @@ def test_FTL_detect_aarch64_no_errors(host): assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv4t_no_errors(host): +def test_FTL_detect_armv4t_no_install(host): """ - confirms only armv4t package is downloaded for FTL engine + confirms armv4t architecture is not supported """ # mock uname to return armv4t platform mock_command("uname", {"-m": ("armv4t", "0")}, host) @@ -288,22 +288,17 @@ def test_FTL_detect_armv4t_no_errors(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" """ ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv4 processor") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" + expected_stdout = cross_box + (" ARM processor without hard-float support detected") assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv5te_no_errors(host): +def test_FTL_detect_armv5te_no_install(host): """ - confirms only armv5te package is downloaded for FTL engine + confirms armv5te architecture is not supported """ # mock uname to return armv5te platform mock_command("uname", {"-m": ("armv5te", "0")}, host) @@ -321,16 +316,11 @@ def test_FTL_detect_armv5te_no_errors(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" """ ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv5 (or newer) processor") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" + expected_stdout = cross_box + (" ARM processor without hard-float support detected") assert expected_stdout in detectPlatform.stdout @@ -375,7 +365,7 @@ def test_FTL_detect_armv7l_no_errors(host): """ # mock uname to return armv7l platform mock_command("uname", {"-m": ("armv7l", "0")}, host) - # mock ldd to respond with ld-linux-armhf shared library + # mock ldd to respond with ld-linux-armhf shared lib rary mock_command( "ldd", { From 366345e87e6dc94294ea746e8cea8f781864a9e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 4 Jun 2023 20:20:18 +0200 Subject: [PATCH 109/462] Fix no_installs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 2 +- test/test_any_automated_install.py | 10 ++-------- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 681b6798..c4ea4764 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2174,7 +2174,7 @@ main() { # this early on as FTL is a hard dependency for Pi-hole local funcOutput funcOutput=$(get_binary_name) #Store output of get_binary_name here - # Abort early if this processor is not supported (get_binary_name returnS empty string) + # Abort early if this processor is not supported (get_binary_name returns empty string) if [[ "${funcOutput}" == "" ]]; then printf " %b Upgrade/install aborted\\n" "${CROSS}" "${DISTRO_NAME}" exit 1 diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 4a86d28e..5c902c2b 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -286,12 +286,9 @@ def test_FTL_detect_armv4t_no_install(host): detectPlatform = host.run( """ source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) + get_binary_name """ ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout expected_stdout = cross_box + (" ARM processor without hard-float support detected") assert expected_stdout in detectPlatform.stdout @@ -314,12 +311,9 @@ def test_FTL_detect_armv5te_no_install(host): detectPlatform = host.run( """ source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) + get_binary_name """ ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout expected_stdout = cross_box + (" ARM processor without hard-float support detected") assert expected_stdout in detectPlatform.stdout From 53fb7ae0ef1b4eefcbf77ed2b4fbaed9a12c55ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 4 Jun 2023 21:06:44 +0200 Subject: [PATCH 110/462] Use shellcheck in smoke-tests. RIP stickler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fca8bb19..d5037f37 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,13 @@ jobs: # If FAIL is 1 then we fail. [[ $FAIL == 1 ]] && exit 1 || echo "Scripts are executable!" + - name: Run shellcheck + uses: ludeeus/action-shellcheck@master + with: + check_together: 'yes' + format: tty + severity: error + - name: Spell-Checking uses: codespell-project/actions-codespell@master with: From ba3e290915a41dbe6a8b885e00ba0a796376fb58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 4 Jun 2023 21:21:48 +0200 Subject: [PATCH 111/462] Fix shellcheck errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeLogFlush.sh | 6 +++--- pihole | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 3473fad5..14542e4b 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -30,10 +30,10 @@ if [ -z "$DBFILE" ]; then DBFILE="/etc/pihole/pihole-FTL.db" fi -if [[ "$@" != *"quiet"* ]]; then +if [[ "$*" != *"quiet"* ]]; then echo -ne " ${INFO} Flushing /var/log/pihole/pihole.log ..." fi -if [[ "$@" == *"once"* ]]; then +if [[ "$*" == *"once"* ]]; then # Nightly logrotation if command -v /usr/sbin/logrotate >/dev/null; then # Logrotate once @@ -69,7 +69,7 @@ else sudo pihole restartdns fi -if [[ "$@" != *"quiet"* ]]; then +if [[ "$*" != *"quiet"* ]]; then echo -e "${OVER} ${TICK} Flushed /var/log/pihole/pihole.log" echo -e " ${TICK} Deleted ${deleted} queries from database" fi diff --git a/pihole b/pihole index 0be995e4..b762c7d4 100755 --- a/pihole +++ b/pihole @@ -196,7 +196,7 @@ Time: echo -e " ${INFO} Blocking already disabled, nothing to do" exit 0 fi - if [[ $# > 1 ]]; then + if [[ $# -gt 1 ]]; then local error=false if [[ "${2}" == *"s" ]]; then tt=${2%"s"} From 088b2c29205c8da2145de664cf2b0978e6fcba79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 1 Jun 2023 15:10:05 +0200 Subject: [PATCH 112/462] Do not remove -all|excat when not sourrounded by space in query.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 1d3b0a29..bfa21247 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -50,7 +50,7 @@ fi # Strip valid options, leaving only the domain and invalid options # This allows users to place the options before or after the domain -options=$(sed -E 's/ ?-(all|exact) ?//g' <<< "${options}") +options=$(sed -E 's/ +-(all|exact) ?//g' <<< "${options}") # Handle remaining options # If $options contain non ASCII characters, convert to punycode From 8bf4ab0cd6ae133661e980a7cd067d23f2bc6b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 7 Jan 2023 21:41:12 +0100 Subject: [PATCH 113/462] Remove option to set static IP address if DHCPCD is installed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 127 ----------------------------- 1 file changed, 127 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index c4ea4764..d49ae012 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -500,9 +500,6 @@ find_IPv4_information() { # disabled as we intentionally want to split on whitespace and have printf populate # the variable with just the first field. printf -v IPv4bare "$(printf ${route#*src })" - # Get the default gateway IPv4 address (the way to reach the Internet) - # shellcheck disable=SC2059,SC2086 - printf -v IPv4gw "$(printf ${route#*via })" if ! valid_ip "${IPv4bare}" ; then IPv4bare="127.0.0.1" @@ -663,135 +660,11 @@ find_IPv6_information() { # A function to collect IPv4 and IPv6 information of the device collect_v4andv6_information() { find_IPv4_information - # Echo the information to the user printf " %b IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}" - # if `dhcpcd` is used offer to set this as static IP for the device - if [[ -f "/etc/dhcpcd.conf" ]]; then - # configure networking via dhcpcd - getStaticIPv4Settings - fi find_IPv6_information printf " %b IPv6 address: %s\\n" "${INFO}" "${IPV6_ADDRESS}" } -getStaticIPv4Settings() { - # Local, named variables - local ipSettingsCorrect - local DHCPChoice - # Ask if the user wants to use DHCP settings as their static IP - # This is useful for users that are using DHCP reservations; we can use the information gathered - DHCPChoice=$(dialog --no-shadow --keep-tite --output-fd 1 \ - --cancel-label "Exit" --ok-label "Continue" \ - --backtitle "Calibrating network interface" \ - --title "Static IP Address" \ - --menu "Do you want to use your current network settings as a static address?\\n \ - IP address: ${IPV4_ADDRESS}\\n \ - Gateway: ${IPv4gw}\\n" \ - "${r}" "${c}" 3 \ - "Yes" "Set static IP using current values" \ - "No" "Set static IP using custom values" \ - "Skip" "I will set a static IP later, or have already done so") - - result=$? - case ${result} in - "${DIALOG_CANCEL}" | "${DIALOG_ESC}") - printf " %b Cancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac - - case ${DHCPChoice} in - "Skip") - return - ;; - "Yes") - # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict. - dialog --no-shadow --keep-tite \ - --cancel-label "Exit" \ - --backtitle "IP information" \ - --title "FYI: IP Conflict" \ - --msgbox "\\nIt is possible your router could still try to assign this IP to a device, which would cause a conflict, \ -but in most cases the router is smart enough to not do that.\n\n\ -If you are worried, either manually set the address, or modify the DHCP reservation pool so it does not include the IP you want.\n\n\ -It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address."\ - "${r}" "${c}" && result=0 || result=$? - - case ${result} in - "${DIALOG_CANCEL}" | "${DIALOG_ESC}") - printf " %b Cancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac - ;; - - "No") - # Otherwise, we need to ask the user to input their desired settings. - # Start by getting the IPv4 address (pre-filling it with info gathered from DHCP) - # Start a loop to let the user enter their information with the chance to go back and edit it if necessary - ipSettingsCorrect=false - until [[ "${ipSettingsCorrect}" = True ]]; do - - # Ask for the IPv4 address - _staticIPv4Temp=$(dialog --no-shadow --keep-tite --output-fd 1 \ - --cancel-label "Exit" \ - --ok-label "Continue" \ - --backtitle "Calibrating network interface" \ - --title "IPv4 Address" \ - --form "\\nEnter your desired IPv4 address" \ - "${r}" "${c}" 0 \ - "IPv4 Address:" 1 1 "${IPV4_ADDRESS}" 1 15 19 0 \ - "IPv4 Gateway:" 2 1 "${IPv4gw}" 2 15 19 0) - - result=$? - case ${result} in - "${DIALOG_CANCEL}" | "${DIALOG_ESC}") - printf " %b Cancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - exit 1 - ;; - esac - - IPV4_ADDRESS=${_staticIPv4Temp%$'\n'*} - IPv4gw=${_staticIPv4Temp#*$'\n'} - - # Give the user a chance to review their settings before moving on - dialog --no-shadow --keep-tite \ - --no-label "Edit IP" \ - --backtitle "Calibrating network interface" \ - --title "Static IP Address" \ - --defaultno \ - --yesno "Are these settings correct? - IP address: ${IPV4_ADDRESS} - Gateway: ${IPv4gw}" \ - "${r}" "${c}" && ipSettingsCorrect=True - done - ;; - esac - setDHCPCD -} - -# Configure networking via dhcpcd -setDHCPCD() { - # Regex for matching a non-commented static ip address setting - local regex="^[ \t]*static ip_address[ \t]*=[ \t]*${IPV4_ADDRESS}" - - # Check if static IP is already set in file - if grep -q "${regex}" /etc/dhcpcd.conf; then - printf " %b Static IP already configured\\n" "${INFO}" - # If it's not, - else - # we can append these lines to dhcpcd.conf to enable a static IP - echo "interface ${PIHOLE_INTERFACE} - static ip_address=${IPV4_ADDRESS} - static routers=${IPv4gw} - static domain_name_servers=${PIHOLE_DNS_1} ${PIHOLE_DNS_2}" | tee -a /etc/dhcpcd.conf >/dev/null - # Then use the ip command to immediately set the new address - ip addr replace dev "${PIHOLE_INTERFACE}" "${IPV4_ADDRESS}" - # Also give a warning that the user may need to reboot their system - printf " %b Set IP address to %s\\n" "${TICK}" "${IPV4_ADDRESS%/*}" - printf " %b You may need to restart after the install is complete\\n" "${INFO}" - fi -} - # Check an IP address to see if it is a valid one valid_ip() { # Local, named variables From 9c27e4766dede7a69b65cc412125ac8186d84852 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 8 Jun 2023 14:10:04 +0200 Subject: [PATCH 114/462] Decide which FTL architecture to pick by by installed architectures rather than by detected processor (this may be wrong in an emulated Docker environment) Signed-off-by: DL6ER --- automated install/basic-install.sh | 76 +++++++---- test/test_any_automated_install.py | 211 ++++++++++++++++++++--------- 2 files changed, 197 insertions(+), 90 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index d49ae012..b14546d1 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1806,10 +1806,19 @@ FTLinstall() { } get_binary_name() { - # This gives the machine architecture which may be different from the OS architecture... + # Get the OS architecture (we cannot use uname -m as this may return an incorrect architecture when buildx-compiling with QEMU for arm) local machine machine=$(uname -m) + # Get local GLIBC version (leave at "0.0" if no GLIBC, e.g., on musl) + local l_glibc_version="0.0" + if ldd --version 2>&1 | grep -q "GLIBC"; then + l_glibc_version=$(ldd --version | head -n1 | grep -o '[0-9.]*$') + printf "%b %b Detected GLIBC version %s\\n" "${OVER}" "${TICK}" "${l_glibc_version}" + else + printf "%b %b No GLIBC detected\\n" "${OVER}" "${CROSS}" + fi + local l_binary local str="Detecting processor" @@ -1817,36 +1826,51 @@ get_binary_name() { # If the machine is arm or aarch if [[ "${machine}" == "arm"* || "${machine}" == *"aarch"* ]]; then # ARM + # Get supported processor from other binaries installed on the system + local cpu_arch + cpu_arch=$(readelf -A "$(command -v sh)" | grep Tag_CPU_arch | awk '{ print $2 }') + + # Get the revision from the CPU architecture local rev - rev=$(uname -m | sed "s/[^0-9]//g;") - local lib - lib=$(ldd "$(command -v sh)" | grep -E '^\s*/lib' | awk '{ print $1 }') - if [[ "${lib}" == "/lib/ld-linux-aarch64.so.1" ]]; then - printf "%b %b Detected AArch64 (64 Bit ARM) processor\\n" "${OVER}" "${TICK}" + rev=$(echo "${cpu_arch}" | grep -o '[0-9]*') + if [[ "${machine}" == "aarch64" ]]; then + printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-arm64" - elif [[ "${lib}" == "/lib/ld-linux-armhf.so.3" ]]; then - # Hard-float available: Use gnueabihf binaries + elif [[ "${cpu_arch}" == "armv6KZ" ]]; then + printf "%b %b Detected ARMv6KZ architecture\\n" "${OVER}" "${TICK}" + # set the binary to be used + l_binary="pihole-FTL-armv6" + else # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) if [[ "${rev}" -gt 7 ]]; then - printf "%b %b Detected ARMv8 (or newer) processor\\n" "${OVER}" "${TICK}" - # set the binary to be used - l_binary="pihole-FTL-armv8" - elif [[ "${rev}" -eq 7 ]]; then - # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) - printf "%b %b Detected ARMv7 processor (with hard-float support)\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv8 (or newer) architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-armv7" - else - # Otherwise, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 processor (with hard-float support)\\n" "${OVER}" "${TICK}" + elif [[ "${rev}" -gt 6 ]]; then + # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) + printf "%b %b Detected ARMv7 architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-armv6" + elif [[ "${rev}" -gt 5 ]]; then + # Check if the system is using GLIBC 2.29 or higher + if [[ -n "${l_glibc_version}" && "$(printf '%s\n' "2.29" "${l_glibc_version}" | sort -V | head -n1)" == "2.29" ]]; then + # If so, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) + printf "%b %b Detected ARMv6 architecture (running GLIBC 2.29 or higher)\\n" "${OVER}" "${TICK}" + # set the binary to be used + l_binary="pihole-FTL-armv5" + else + # Otherwise, use the ARMv5 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) + printf "%b %b Detected ARMv6 architecture (running GLIBC older than 2.29)\\n" "${OVER}" "${TICK}" + # set the binary to be used + l_binary="pihole-FTL-armv4" + fi + else + # Otherwise, use the ARMv4 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) + printf "%b %b Detected ARMv4 or ARMv5 architecture\\n" "${OVER}" "${TICK}" + # set the binary to be used + l_binary="pihole-FTL-armv4" fi - else - # No hard-float support found - printf "%b %b%b ARM processor without hard-float support detected%b\\n" "${OVER}" "${COL_LIGHT_RED}" "${CROSS}" "${COL_NC}" - l_binary="" fi elif [[ "${machine}" == "x86_64" ]]; then # This gives the processor of packages dpkg installs (for example, "i386") @@ -1858,25 +1882,25 @@ get_binary_name() { # We only check this for Debian-based systems as this has been an issue # in the past (see https://github.com/pi-hole/pi-hole/pull/2004) if [[ "${dpkgarch}" == "i386" ]]; then - printf "%b %b Detected 32bit (i686) processor\\n" "${OVER}" "${TICK}" + printf "%b %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-386" else # 64bit - printf "%b %b Detected x86_64 processor\\n" "${OVER}" "${TICK}" + printf "%b %b Detected x86_64 architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-amd64" fi elif [[ "${machine}" == "riscv64" ]]; then - printf "%b %b Detected riscv64 processor\\n" "${OVER}" "${TICK}" + printf "%b %b Detected riscv64 architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-riscv64" else # Something else - we try to use 32bit executable and warn the user if [[ ! "${machine}" == "i686" ]]; then printf "%b %b %s...\\n" "${OVER}" "${CROSS}" "${str}" - printf " %b %bNot able to detect processor (unknown: %s), trying x86 (32bit) executable%b\\n" "${INFO}" "${COL_LIGHT_RED}" "${machine}" "${COL_NC}" + printf " %b %bNot able to detect architecture (unknown: %s), trying x86 (32bit) executable%b\\n" "${INFO}" "${COL_LIGHT_RED}" "${machine}" "${COL_NC}" printf " %b Contact Pi-hole Support if you experience issues (e.g: FTL not running)\\n" "${INFO}" else - printf "%b %b Detected 32bit (i686) processor\\n" "${OVER}" "${TICK}" + printf "%b %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}" fi l_binary="pihole-FTL-linux-386" fi diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 5c902c2b..4431c565 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -241,12 +241,36 @@ def test_FTL_detect_aarch64_no_errors(host): """ # mock uname to return aarch64 platform mock_command("uname", {"-m": ("aarch64", "0")}, host) - # mock ldd to respond with aarch64 shared library - mock_command( - "ldd", + detectPlatform = host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Detected AArch64 (64 Bit ARM) architecture" + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + + +def test_FTL_detect_armv4_no_errors(host): + """ + confirms only armv4 package is downloaded for FTL engine + """ + # mock uname to return armv4 platform + mock_command("uname", {"-m": ("armv4t", "0")}, host) + # mock readelf to respond with armv4 CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux-aarch64.so.1", "0"), - "/usr/bin/sh": ("/lib/ld-linux-aarch64.so.1", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv4t", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv4t", "0"), }, host, ) @@ -262,77 +286,65 @@ def test_FTL_detect_aarch64_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected AArch64 (64 Bit ARM) processor" + expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv4t_no_install(host): +def test_FTL_detect_armv5_no_errors(host): """ - confirms armv4t architecture is not supported - """ - # mock uname to return armv4t platform - mock_command("uname", {"-m": ("armv4t", "0")}, host) - # mock ldd to respond with armv4t shared library - mock_command( - "ldd", - { - "/bin/sh": ("/lib/ld-linux.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux.so.3", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - get_binary_name - """ - ) - expected_stdout = cross_box + (" ARM processor without hard-float support detected") - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv5te_no_install(host): - """ - confirms armv5te architecture is not supported + confirms only armv5 package is downloaded for FTL engine """ # mock uname to return armv5te platform mock_command("uname", {"-m": ("armv5te", "0")}, host) - # mock ldd to respond with ld-linux shared library - mock_command( - "ldd", + # mock readelf to respond with armv5 CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux.so.3", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv5te", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv5te", "0"), }, host, ) detectPlatform = host.run( """ source /opt/pihole/basic-install.sh - get_binary_name + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" """ ) - expected_stdout = cross_box + (" ARM processor without hard-float support detected") + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture" + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv6l_no_errors(host): +def test_FTL_detect_armv6_old_no_errors(host): """ - confirms only armv6l package is downloaded for FTL engine + confirms only armv6 package is downloaded for FTL engine """ # mock uname to return armv6l platform mock_command("uname", {"-m": ("armv6l", "0")}, host) - # mock ldd to respond with ld-linux-armhf shared library - mock_command( - "ldd", + # mock readelf to respond with armv6l CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv6l", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv6l", "0"), }, host, ) + # Mock old ldd GLIBC version + mock_command( + "ldd", {"--version": ("ldd (Debian GLIBC 2.13-38+deb7u8) 2.13", "0")}, host + ) + detectPlatform = host.run( """ source /opt/pihole/basic-install.sh @@ -346,25 +358,98 @@ def test_FTL_detect_armv6l_no_errors(host): expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + ( - " Detected ARMv6 processor " "(with hard-float support)" + " Detected ARMv6 architecture (running GLIBC older than 2.29)" ) assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout +def test_FTL_detect_armv6_recent_no_errors(host): + """ + confirms only armv6 package is downloaded for FTL engine + """ + # mock uname to return armv6l platform + mock_command("uname", {"-m": ("armv6l", "0")}, host) + # mock readelf to respond with armv6l CPU architecture + mock_command_2( + "readelf", + { + "-A /bin/sh": ("Tag_CPU_arch: armv6l", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv6l", "0"), + }, + host, + ) + # Mock old ldd GLIBC version + mock_command( + "ldd", {"--version": ("'ldd (Debian GLIBC 2.35-38+deb7u8) 2.35'", "0")}, host + ) + + detectPlatform = host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + ( + " Detected ARMv6 architecture (running GLIBC 2.29 or higher)" + ) + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + + +def test_FTL_detect_armv6KZ_no_errors(host): + """ + confirms only armv6KZ package is downloaded for FTL engine + """ + # mock uname to return armv6KZ platform + mock_command("uname", {"-m": ("armv6KZ", "0")}, host) + # mock readelf to respond with armv6l CPU architecture + mock_command_2( + "readelf", + { + "-A /bin/sh": ("Tag_CPU_arch: armv6KZ", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv6KZ", "0"), + }, + host, + ) + detectPlatform = host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Detected ARMv6KZ architecture" + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + + def test_FTL_detect_armv7l_no_errors(host): """ confirms only armv7l package is downloaded for FTL engine """ # mock uname to return armv7l platform mock_command("uname", {"-m": ("armv7l", "0")}, host) - # mock ldd to respond with ld-linux-armhf shared lib rary - mock_command( - "ldd", + # mock readelf to respond with armv7l CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv7l", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv7l", "0"), }, host, ) @@ -380,9 +465,7 @@ def test_FTL_detect_armv7l_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + ( - " Detected ARMv7 processor " "(with hard-float support)" - ) + expected_stdout = tick_box + (" Detected ARMv7 architecture") assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -394,12 +477,12 @@ def test_FTL_detect_armv8a_no_errors(host): """ # mock uname to return armv8a platform mock_command("uname", {"-m": ("armv8a", "0")}, host) - # mock ldd to respond with ld-linux-armhf shared library - mock_command( - "ldd", + # mock readelf to respond with armv8a CPU architecture + mock_command_2( + "readelf", { - "/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), - "/usr/bin/sh": ("/lib/ld-linux-armhf.so.3", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv8a", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv8a", "0"), }, host, ) @@ -415,7 +498,7 @@ def test_FTL_detect_armv8a_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv8 (or newer) processor" + expected_stdout = tick_box + " Detected ARMv8 (or newer) architecture" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -437,7 +520,7 @@ def test_FTL_detect_x86_64_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected x86_64 processor" + expected_stdout = tick_box + " Detected x86_64 architecture" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -457,7 +540,7 @@ def test_FTL_detect_unknown_no_errors(host): FTLdetect "${binary}" "${theRest}" """ ) - expected_stdout = "Not able to detect processor (unknown: mips)" + expected_stdout = "Not able to detect architecture (unknown: mips)" assert expected_stdout in detectPlatform.stdout From 1c4e58efe3487264291761ec48a0dd0c7d61417e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Jun 2023 15:53:02 +0200 Subject: [PATCH 115/462] Use exact architecture matching binaries where appropriate Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index b14546d1..ed2d0037 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1843,7 +1843,7 @@ get_binary_name() { l_binary="pihole-FTL-armv6" else # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) - if [[ "${rev}" -gt 7 ]]; then + if [[ "${cpu_arch}" == "v7" || "${rev}" -gt 7 ]]; then printf "%b %b Detected ARMv8 (or newer) architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-armv7" @@ -1852,7 +1852,7 @@ get_binary_name() { printf "%b %b Detected ARMv7 architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-armv6" - elif [[ "${rev}" -gt 5 ]]; then + elif [[ "${cpu_arch}" == "v5TE" || "${rev}" -gt 5 ]]; then # Check if the system is using GLIBC 2.29 or higher if [[ -n "${l_glibc_version}" && "$(printf '%s\n' "2.29" "${l_glibc_version}" | sort -V | head -n1)" == "2.29" ]]; then # If so, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) From 913be6c3494db826ee0914034bb42d479c4ec8c9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Jun 2023 18:50:31 +0200 Subject: [PATCH 116/462] Print detected CPU arch Signed-off-by: DL6ER --- automated install/basic-install.sh | 10 +++---- test/test_any_automated_install.py | 45 ++++++++++++++++++++++++++---- 2 files changed, 44 insertions(+), 11 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ed2d0037..5a6be794 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1844,30 +1844,30 @@ get_binary_name() { else # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) if [[ "${cpu_arch}" == "v7" || "${rev}" -gt 7 ]]; then - printf "%b %b Detected ARMv8 (or newer) architecture\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv7 (or newer) architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv7" elif [[ "${rev}" -gt 6 ]]; then # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) - printf "%b %b Detected ARMv7 architecture\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv6" elif [[ "${cpu_arch}" == "v5TE" || "${rev}" -gt 5 ]]; then # Check if the system is using GLIBC 2.29 or higher if [[ -n "${l_glibc_version}" && "$(printf '%s\n' "2.29" "${l_glibc_version}" | sort -V | head -n1)" == "2.29" ]]; then # If so, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 architecture (running GLIBC 2.29 or higher)\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv6 architecture (running GLIBC 2.29 or higher, %s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv5" else # Otherwise, use the ARMv5 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 architecture (running GLIBC older than 2.29)\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv6 architecture (running GLIBC older than 2.29, %s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv4" fi else # Otherwise, use the ARMv4 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv4 or ARMv5 architecture\\n" "${OVER}" "${TICK}" + printf "%b %b Detected ARMv4 or ARMv5 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv4" fi diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 4431c565..bc594304 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -286,7 +286,7 @@ def test_FTL_detect_armv4_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture" + expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture (armv4t)" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -319,7 +319,7 @@ def test_FTL_detect_armv5_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture" + expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture (armv5te)" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -358,7 +358,7 @@ def test_FTL_detect_armv6_old_no_errors(host): expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + ( - " Detected ARMv6 architecture (running GLIBC older than 2.29)" + " Detected ARMv6 architecture (running GLIBC older than 2.29, armv6l)" ) assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" @@ -398,7 +398,7 @@ def test_FTL_detect_armv6_recent_no_errors(host): expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + ( - " Detected ARMv6 architecture (running GLIBC 2.29 or higher)" + " Detected ARMv6 architecture (running GLIBC 2.29 or higher, armv6l)" ) assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" @@ -465,7 +465,40 @@ def test_FTL_detect_armv7l_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 architecture") + expected_stdout = tick_box + (" Detected ARMv7 architecture (armv7l)") + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + + +def test_FTL_detect_armv7_no_errors(host): + """ + confirms only armv7 package is downloaded for FTL engine + """ + # mock uname to return armv7 platform + mock_command("uname", {"-m": ("armv7", "0")}, host) + # mock readelf to respond with armv7 CPU architecture + mock_command_2( + "readelf", + { + "-A /bin/sh": ("Tag_CPU_arch: armv7", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv7", "0"), + }, + host, + ) + detectPlatform = host.run( + """ + source /opt/pihole/basic-install.sh + create_pihole_user + funcOutput=$(get_binary_name) + binary="pihole-FTL${funcOutput##*pihole-FTL}" + theRest="${funcOutput%pihole-FTL*}" + FTLdetect "${binary}" "${theRest}" + """ + ) + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + (" Detected ARMv7 architecture (armv7)") assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -498,7 +531,7 @@ def test_FTL_detect_armv8a_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv8 (or newer) architecture" + expected_stdout = tick_box + " Detected ARMv7 (or newer) architecture (armv8a)" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout From e0d7e5df8519883cf08b45500809ef1d04ee4160 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 9 Jun 2023 19:30:28 +0200 Subject: [PATCH 117/462] Install binutils as installer deps (needed to pick correct architecture for FTL) Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 5a6be794..699fef10 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -304,7 +304,7 @@ package_manager_detect() { # Packages required to perform the os_check (stored as an array) OS_CHECK_DEPS=(grep dnsutils) # Packages required to run this install script (stored as an array) - INSTALLER_DEPS=(git iproute2 dialog ca-certificates) + INSTALLER_DEPS=(git iproute2 dialog ca-certificates binutils) # Packages required to run Pi-hole (stored as an array) PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) @@ -322,7 +322,7 @@ package_manager_detect() { # CentOS package manager returns 100 when there are packages to update so we need to || true to prevent the script from exiting. PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) - INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates) + INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat jq) # If neither apt-get or yum/dnf package managers were found From 7d3f354dd71c9cc0fc3abe7e0ad401ad8e864aae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Jun 2023 10:57:12 +0000 Subject: [PATCH 118/462] Bump actions/checkout from 3.5.2 to 3.5.3 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.5.2 to 3.5.3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.5.2...v3.5.3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index f3bcf15a..7fd0e4e4 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index fe28112c..43d5ca96 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index fa7564a3..5435b7db 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fca8bb19..6cb88d99 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: Check scripts in repository are executable run: | @@ -64,7 +64,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.5.2 + uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 uses: actions/setup-python@v4.6.1 From 8e481e27da2c3bf47edcb7f26b92c06300dd7f16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Jun 2023 11:00:35 +0000 Subject: [PATCH 119/462] Bump tox from 4.5.2 to 4.6.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.5.2 to 4.6.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.5.2...4.6.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index de54d22c..035ce805 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.3.1 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.5.2 +tox == 4.6.0 From 117c15319da67de4e9bf35ba25d7cc0d8996291a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 10 Jun 2023 23:03:43 +0200 Subject: [PATCH 120/462] Add tests for Debian 12 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 1 + test/_debian_12.Dockerfile | 17 +++++++++++++++++ test/tox.debian_12.ini | 8 ++++++++ 3 files changed, 26 insertions(+) create mode 100644 test/_debian_12.Dockerfile create mode 100644 test/tox.debian_12.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6cb88d99..c9184615 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -51,6 +51,7 @@ jobs: [ debian_10, debian_11, + debian_12, ubuntu_20, ubuntu_22, ubuntu_23, diff --git a/test/_debian_12.Dockerfile b/test/_debian_12.Dockerfile new file mode 100644 index 00000000..a762fee0 --- /dev/null +++ b/test/_debian_12.Dockerfile @@ -0,0 +1,17 @@ +FROM buildpack-deps:bookworm-scm + +ENV GITDIR /etc/.pihole +ENV SCRIPTDIR /opt/pihole + +RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole +ADD . $GITDIR +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR + +RUN true && \ + chmod +x $SCRIPTDIR/* + +ENV SKIP_INSTALL true +ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net + +#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.debian_12.ini b/test/tox.debian_12.ini new file mode 100644 index 00000000..ee70e8bd --- /dev/null +++ b/test/tox.debian_12.ini @@ -0,0 +1,8 @@ +[tox] +envlist = py3 + +[testenv:py3] +allowlist_externals = docker +deps = -rrequirements.txt +commands = docker buildx build --load --progress plain -f _debian_12.Dockerfile -t pytest_pihole:test_container ../ + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py From 5459daa03daaaf89e18ac3da64082193c9c2b0c4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 15 Jun 2023 09:28:03 +0200 Subject: [PATCH 121/462] Add abp_entries column (default 0) to the adlist table for newly created gravity databases. This updates the gravity database version to 16 Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 6 ++++++ .../Scripts/database_migration/gravity/15_to_16.sql | 11 +++++++++++ advanced/Templates/gravity.db.sql | 3 ++- 3 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 advanced/Scripts/database_migration/gravity/15_to_16.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index a7ba60a9..851ddb0a 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -128,4 +128,10 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" version=15 fi + if [[ "$version" == "15" ]]; then + # Add column abp_entries to adlist table + echo -e " ${INFO} Upgrading gravity database from version 15 to 16" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/15_to_16.sql" + version=16 + fi } diff --git a/advanced/Scripts/database_migration/gravity/15_to_16.sql b/advanced/Scripts/database_migration/gravity/15_to_16.sql new file mode 100644 index 00000000..c6159f40 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/15_to_16.sql @@ -0,0 +1,11 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +ALTER TABLE adlist ADD COLUMN abp_entries INTEGER NOT NULL DEFAULT 0; + +UPDATE info SET value = 16 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 3f696d6d..e0f1de10 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -35,7 +35,8 @@ CREATE TABLE adlist date_updated INTEGER, number INTEGER NOT NULL DEFAULT 0, invalid_domains INTEGER NOT NULL DEFAULT 0, - status INTEGER NOT NULL DEFAULT 0 + status INTEGER NOT NULL DEFAULT 0, + abp_entries INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE adlist_by_group From 3fd7b4ee243e84d371eb876cef52186a44c2bd6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Jun 2023 10:59:42 +0000 Subject: [PATCH 122/462] Bump pytest from 7.3.1 to 7.3.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.3.1 to 7.3.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.3.1...7.3.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 035ce805..407c3e3c 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ docker-compose == 1.29.2 -pytest == 7.3.1 +pytest == 7.3.2 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 tox == 4.6.0 From 7a66083e6888d2109eaa9b042694cf1a1204adad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Jun 2023 11:35:38 +0000 Subject: [PATCH 123/462] Bump tox from 4.6.0 to 4.6.2 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.6.0 to 4.6.2. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.6.0...4.6.2) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 407c3e3c..44e6c94a 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.3.2 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.6.0 +tox == 4.6.2 From d5013bfd6c3153113fa8a67b6731c9d91a327e67 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 12 Jun 2023 22:33:50 +0100 Subject: [PATCH 124/462] Add code to remove old lighttpd config files left over from v5. Web config is all dealt with by FTL now Signed-off-by: Adam Warner --- automated install/basic-install.sh | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 699fef10..9659a140 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1001,6 +1001,38 @@ remove_old_dnsmasq_ftl_configs() { fi } +remove_old_pihole_lighttpd_configs() { + local lighttpdConfig="/etc/lighttpd/lighttpd.conf" + local condfd="/etc/lighttpd/conf.d/pihole-admin.conf" + local confavailable="/etc/lighttpd/conf-available/15-pihole-admin.conf" + local confenabled="/etc/lighttpd/conf-enabled/15-pihole-admin.conf" + + + if [[ -d "/etc/lighttpd/conf.d" ]]; then + if grep -q -F 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' "${lighttpdConfig}"; then + sed -i '/include "/etc/lighttpd/conf.d/pihole-admin.conf"/d' "${lighttpdConfig}" + fi + + if [[ -f "${condfd}" ]]; then + rm "${condfd}" + fi + + + elif [[ -d "/etc/lighttpd/conf-available" ]]; then + if is_command lighty-disable-mod ; then + lighty-disable-mod pihole-admin > /dev/null || true + fi + + if [[ -f "${confavailable}" ]]; then + rm "${confavailable}" + fi + + if [[ -f "${confenabled}" ]]; then + rm "${confenabled}" + fi + fi +} + # Clean an existing installation to prepare for upgrade/reinstall clean_existing() { # Local, named variables @@ -1486,6 +1518,7 @@ installPihole() { fi remove_old_dnsmasq_ftl_configs + remove_old_pihole_lighttpd_configs # Install config files if ! installConfigs; then From c39cb8cfe066d3520354f3cdee293fa6afe47e0e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 12 Jun 2023 22:43:39 +0100 Subject: [PATCH 125/462] Escape the sed command for removing a line in fed/centos lighttpd.conf Signed-off-by: Adam Warner --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9659a140..43cd9816 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1010,7 +1010,7 @@ remove_old_pihole_lighttpd_configs() { if [[ -d "/etc/lighttpd/conf.d" ]]; then if grep -q -F 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' "${lighttpdConfig}"; then - sed -i '/include "/etc/lighttpd/conf.d/pihole-admin.conf"/d' "${lighttpdConfig}" + sed -i '/include "\/etc\/lighttpd\/conf.d\/pihole-admin.conf"/d' "${lighttpdConfig}" fi if [[ -f "${condfd}" ]]; then From 001f2012a2b40c200e4c7d6384b8cd2f62c523c5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 13 Jun 2023 19:08:12 +0100 Subject: [PATCH 126/462] Update automated install/basic-install.sh Co-authored-by: RD WebDesign Signed-off-by: Adam Warner --- automated install/basic-install.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 43cd9816..2edad6e3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1009,9 +1009,7 @@ remove_old_pihole_lighttpd_configs() { if [[ -d "/etc/lighttpd/conf.d" ]]; then - if grep -q -F 'include "/etc/lighttpd/conf.d/pihole-admin.conf"' "${lighttpdConfig}"; then - sed -i '/include "\/etc\/lighttpd\/conf.d\/pihole-admin.conf"/d' "${lighttpdConfig}" - fi + sed -i '/include "\/etc\/lighttpd\/conf.d\/pihole-admin.conf"/d' "${lighttpdConfig}" if [[ -f "${condfd}" ]]; then rm "${condfd}" From d637d2a7a5f76fdea076d01d091e9e4865ac55f4 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 18 Jun 2023 12:38:02 +0100 Subject: [PATCH 127/462] Simplify nested if statements. Co-authored-by: yubiuser Signed-off-by: Adam Warner --- automated install/basic-install.sh | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 2edad6e3..e3d8ff29 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1008,27 +1008,25 @@ remove_old_pihole_lighttpd_configs() { local confenabled="/etc/lighttpd/conf-enabled/15-pihole-admin.conf" - if [[ -d "/etc/lighttpd/conf.d" ]]; then + if [[ -f "${lighttpdConfig}" ]]; then sed -i '/include "\/etc\/lighttpd\/conf.d\/pihole-admin.conf"/d' "${lighttpdConfig}" + fi - if [[ -f "${condfd}" ]]; then - rm "${condfd}" - fi + if [[ -f "${condfd}" ]]; then + rm "${condfd}" + fi + if is_command lighty-disable-mod ; then + lighty-disable-mod pihole-admin > /dev/null || true + fi - elif [[ -d "/etc/lighttpd/conf-available" ]]; then - if is_command lighty-disable-mod ; then - lighty-disable-mod pihole-admin > /dev/null || true - fi + if [[ -f "${confavailable}" ]]; then + rm "${confavailable}" + fi - if [[ -f "${confavailable}" ]]; then - rm "${confavailable}" - fi - - if [[ -f "${confenabled}" ]]; then - rm "${confenabled}" - fi - fi + if [[ -f "${confenabled}" ]]; then + rm "${confenabled}" + fi } # Clean an existing installation to prepare for upgrade/reinstall From a3bb3872bfbf9520bd29b2069a889d7d6d4f1202 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 18 Jun 2023 16:30:59 +0100 Subject: [PATCH 128/462] adlist table now contains 11 columns, not 10. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Christian König Signed-off-by: Adam Warner --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index abf6700b..ed402a34 100755 --- a/gravity.sh +++ b/gravity.sh @@ -178,7 +178,7 @@ database_table_from_file() { echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" elif [[ "${table}" == "adlist" ]]; then # Adlist table format - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0" >> "${tmpFile}" else # White-, black-, and regexlist table format echo "${rowid},${list_type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\"" >> "${tmpFile}" From ca66c1ea9cd905b06618dbded58ff49b9b4f45a1 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 18 Jun 2023 18:01:54 +0100 Subject: [PATCH 129/462] Switch to new branch name for FTL v6 development Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 2 +- test/test_any_utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index bc594304..a8e0e082 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -82,7 +82,7 @@ def test_installPihole_fresh_install_readableFiles(host): # Workaround to get FTLv6 installed until it reaches master branch host.run( """ - echo "new/http" > /etc/pihole/ftlbranch + echo "development-v6" > /etc/pihole/ftlbranch """ ) install = host.run( diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 0f300457..9eee6885 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -137,7 +137,7 @@ def test_getFTLConfigValue_getFTLConfigValue(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - echo "new/http" > /etc/pihole/ftlbranch + echo "development-v6" > /etc/pihole/ftlbranch binary="pihole-FTL${funcOutput##*pihole-FTL}" theRest="${funcOutput%pihole-FTL*}" FTLdetect "${binary}" "${theRest}" From 198ca65f7debc376fc8178483ed50c1047aa3129 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Jun 2023 10:57:28 +0000 Subject: [PATCH 130/462] Bump tox from 4.6.2 to 4.6.3 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.6.2 to 4.6.3. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.6.2...4.6.3) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 44e6c94a..67dbcf68 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.3.2 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.6.2 +tox == 4.6.3 From 5490625d8d731347efc968d5187723c62ad3cee6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Jun 2023 16:03:56 +0000 Subject: [PATCH 131/462] Bump pytest from 7.3.2 to 7.4.0 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.3.2 to 7.4.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.3.2...7.4.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 67dbcf68..a08dff94 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ docker-compose == 1.29.2 -pytest == 7.3.2 +pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 tox == 4.6.3 From c557f29db2d8d222fdd9d38c5b53fcfac110aeb8 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 3 Jul 2023 23:01:27 +0100 Subject: [PATCH 132/462] Set new gravity database version to 16, which was missed in 5459daa Signed-off-by: Adam Warner --- advanced/Templates/gravity.db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index e0f1de10..881cfcc3 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -58,7 +58,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','15'); +INSERT INTO "info" VALUES('version','16'); CREATE TABLE domain_audit ( From 64319fa96efcd1b8172afb85110ae0742d49185c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 5 Jul 2023 22:11:56 +0100 Subject: [PATCH 133/462] Add /var/log/pihole/webserver.log to the logrotate scripts. While it may be empty for most, it can grow quickly if either API or TLS debugging is enabled (it can quickly reach several hundreds of MB per day when debugging TLS) Signed-off-by: DL6ER --- advanced/Templates/logrotate | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/advanced/Templates/logrotate b/advanced/Templates/logrotate index 9a56b552..5f609e0f 100644 --- a/advanced/Templates/logrotate +++ b/advanced/Templates/logrotate @@ -19,3 +19,14 @@ notifempty nomail } + +/var/log/pihole/webserver.log { + # su # + weekly + copytruncate + rotate 3 + compress + delaycompress + notifempty + nomail +} From a01d31e25da2cb20ceac18e5b1ecc0916f27b902 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 8 Jul 2023 10:15:54 +0000 Subject: [PATCH 134/462] Bump tox from 4.6.3 to 4.6.4 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.6.3 to 4.6.4. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.6.3...4.6.4) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index a08dff94..a4f627fa 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.6.3 +tox == 4.6.4 From 924de1d9ed9f18bcef484f1aa87961b267c562d7 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 11 Jul 2023 23:51:28 +0100 Subject: [PATCH 135/462] Update dependabot.yml Signed-off-by: Adam Warner --- .github/dependabot.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e140f792..20163f5e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -20,3 +20,24 @@ updates: target-branch: development reviewers: - "pi-hole/core-maintainers" +# As above, but for development-v6 +- package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + day: saturday + time: "10:00" + open-pull-requests-limit: 10 + target-branch: development-v6 + reviewers: + - "pi-hole/core-maintainers" +- package-ecosystem: pip + directory: "/test" + schedule: + interval: weekly + day: saturday + time: "10:00" + open-pull-requests-limit: 10 + target-branch: development-v6 + reviewers: + - "pi-hole/core-maintainers" From 80091232a767efd28277e90e008c643266e06577 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 12 Jul 2023 21:31:19 +0100 Subject: [PATCH 136/462] Add missing `env:`, tweak conditional Signed-off-by: Adam Warner --- .github/workflows/stale.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 0d06ee22..458a7fb6 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -32,8 +32,8 @@ jobs: remove_stale: # trigger "stale" removal immediately when stale issues are commented on # we need to explicitly check that the trigger does not run on comment on a PR as - # 'issue_comment' triggers on issues AND PR comments - if: github.event_name == 'issue_comment' && ${{ !github.event.issue.pull_request }} + # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#issue_comment-on-issues-only-or-pull-requests-only + if: ${{ !github.event.issue.pull_request && github.event_name != 'schedule' }} permissions: contents: read # for actions/checkout issues: write # to edit issues label @@ -43,5 +43,6 @@ jobs: uses: actions/checkout@v3.5.2 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 91dabc574d8d9c38379ec4db710f3799be55a8fc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Jul 2023 10:30:37 +0000 Subject: [PATCH 137/462] Bump actions/setup-python from 4.6.1 to 4.7.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.6.1 to 4.7.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.6.1...v4.7.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c9184615..8546e02b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 - uses: actions/setup-python@v4.6.1 + uses: actions/setup-python@v4.7.0 with: python-version: "3.10" From e295997d51d50f661e05eee06b290ad57a56f502 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 21:36:02 +0000 Subject: [PATCH 138/462] Bump actions/setup-python from 4.6.1 to 4.7.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.6.1 to 4.7.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.6.1...v4.7.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index be64482d..09462eb4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: uses: actions/checkout@v3.5.3 - name: Set up Python 3.10 - uses: actions/setup-python@v4.6.1 + uses: actions/setup-python@v4.7.0 with: python-version: "3.10" From a3955a7ebb33cdf1ec3ab537eb2a507705648c55 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 21:36:15 +0000 Subject: [PATCH 139/462] Bump pytest from 7.3.2 to 7.4.0 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.3.2 to 7.4.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.3.2...7.4.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 44e6c94a..4eb9113e 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ docker-compose == 1.29.2 -pytest == 7.3.2 +pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 tox == 4.6.2 From 8dd8f989abce529ed3176eeecc7174c8ed3a9053 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 16 Jul 2023 21:44:31 +0000 Subject: [PATCH 140/462] Bump tox from 4.6.2 to 4.6.4 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.6.2 to 4.6.4. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.6.2...4.6.4) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 4eb9113e..a4f627fa 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ docker-compose == 1.29.2 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.6.2 +tox == 4.6.4 From 7e240251d55ea5830260fc9d3bc1106d1e33bdb2 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 18 Jul 2023 08:03:05 +0100 Subject: [PATCH 141/462] Drop docker-compose from requirements.txt, it's not used. This was the only thing preventing us from using pyyaml 6.0.1 which fixes issues we were seeing with tests not running (https://github.com/yaml/pyyaml/pull/702) Signed-off-by: Adam Warner --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index a4f627fa..5a551fa7 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,4 +1,4 @@ -docker-compose == 1.29.2 +pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 From e42c692ec7b364d443c947187f7ab5943f6f6173 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 18 Jul 2023 08:04:33 +0100 Subject: [PATCH 142/462] Drop docker-compose from requirements.txt, it's not used. This was the only thing preventing us from using pyyaml 6.0.1 which fixes issues we were seeing with tests not running (https://github.com/yaml/pyyaml/pull/702) Signed-off-by: Adam Warner --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index a4f627fa..5a551fa7 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,4 +1,4 @@ -docker-compose == 1.29.2 +pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 From 6edd6a4d55af1ca29c799c96ee355f0747df1ea7 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 17 Jul 2023 21:44:13 +0100 Subject: [PATCH 143/462] if ${USER} variable is blank, then populate it with whoami Signed-off-by: Adam Warner --- pihole | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pihole b/pihole index b762c7d4..66771b9b 100755 --- a/pihole +++ b/pihole @@ -544,6 +544,13 @@ if [[ ! $EUID -eq 0 && need_root -eq 1 ]];then exit 1 fi fi + +# In the case of alpine running in a container, the USER variable appears to be blank +# which prevents the next trap from working correctly. Set it by running whoami +if [[ -z ${USER} ]]; then + USER=$(whoami) +fi + # Can also be user pihole for other functions if [[ ${USER} != "pihole" && need_root -eq 0 ]];then if [[ -x "$(command -v sudo)" ]]; then From 00cbb8bc8abb119ebc3c3f0d912b94a334e06128 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 5 Jul 2023 22:24:11 +0200 Subject: [PATCH 144/462] Add antigravity support to gravity Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 7 +++++ .../database_migration/gravity/16_to_17.sql | 13 ++++++++++ advanced/Templates/gravity.db.sql | 16 +++++++++++- gravity.sh | 26 +++++++++++++------ 4 files changed, 53 insertions(+), 9 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/16_to_17.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 851ddb0a..e36d9b1e 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -134,4 +134,11 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 "${database}" < "${scriptPath}/15_to_16.sql" version=16 fi + if [[ "$version" == "16" ]]; then + # Add antigravity table + # Add column type to adlist table (to support adlist types) + echo -e " ${INFO} Upgrading gravity database from version 16 to 17" + pihole-FTL sqlite3 "${database}" < "${scriptPath}/16_to_17.sql" + version=17 + fi } diff --git a/advanced/Scripts/database_migration/gravity/16_to_17.sql b/advanced/Scripts/database_migration/gravity/16_to_17.sql new file mode 100644 index 00000000..c7b9049b --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/16_to_17.sql @@ -0,0 +1,13 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +ALTER TABLE adlist ADD COLUMN type INTEGER NOT NULL DEFAULT 0; + +UPDATE adlist SET type = 0; + +UPDATE info SET value = 17 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 881cfcc3..17712cf7 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -36,7 +36,8 @@ CREATE TABLE adlist number INTEGER NOT NULL DEFAULT 0, invalid_domains INTEGER NOT NULL DEFAULT 0, status INTEGER NOT NULL DEFAULT 0, - abp_entries INTEGER NOT NULL DEFAULT 0 + abp_entries INTEGER NOT NULL DEFAULT 0, + type INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE adlist_by_group @@ -52,6 +53,12 @@ CREATE TABLE gravity adlist_id INTEGER NOT NULL REFERENCES adlist (id) ); +CREATE TABLE antigravity +( + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id) +); + CREATE TABLE info ( property TEXT PRIMARY KEY, @@ -144,6 +151,13 @@ CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); +CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + CREATE VIEW vw_adlist AS SELECT DISTINCT address, id FROM adlist WHERE enabled = 1 diff --git a/gravity.sh b/gravity.sh index ed402a34..d784a2e7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -361,6 +361,7 @@ gravity_DownloadBlocklists() { # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceTypes <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( @@ -382,7 +383,7 @@ gravity_DownloadBlocklists() { unset sources fi - local url domain agent str target compression + local url domain agent str target compression adlist_type echo "" # Prepare new gravity database @@ -394,7 +395,7 @@ gravity_DownloadBlocklists() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to create new database ${gravityTEMPfile}\\n ${output}" - gravity_Cleanup "error" + #gravity_Cleanup "error" else echo -e "${OVER} ${TICK} ${str}" fi @@ -433,6 +434,15 @@ gravity_DownloadBlocklists() { url="${sources[$i]}" domain="${sourceDomains[$i]}" id="${sourceIDs[$i]}" + if [[ "${sourceTypes[$i]}" -eq "0" ]]; then + # Gravity list + str="blocklist" + adlist_type="gravity" + else + # AntiGravity list + str="allowlist" + adlist_type="antigravity" + fi # Save the file as list.#.domain saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}" @@ -441,7 +451,7 @@ gravity_DownloadBlocklists() { # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-) agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" - echo -e " ${INFO} Target: ${url}" + echo -e " ${INFO} Target: ${url} (${str})" local regex check_url # Check for characters NOT allowed in URLs regex="[^a-zA-Z0-9:/?&%=~._()-;]" @@ -453,7 +463,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" + gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" fi echo "" done @@ -485,7 +495,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" + local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" gravity_type="${7}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM @@ -579,7 +589,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table file - pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "2" done="true" # Check if $listCurlBuffer is a non-zero length file @@ -589,7 +599,7 @@ gravity_DownloadBlocklistFromUrl() { # Remove curl buffer file after its use rm "${listCurlBuffer}" # Add domains to database table file - pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" # Compare lists, are they identical? compareLists "${adlistID}" "${saveLocation}" done="true" @@ -605,7 +615,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" # Add domains to database table file - pihole-FTL gravity parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" From 35512c4dc9071502b10f251d514d297a79fe7ea5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jul 2023 19:19:55 +0200 Subject: [PATCH 145/462] Fix adlist.list migration step failing during tests Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index d784a2e7..89b4ab84 100755 --- a/gravity.sh +++ b/gravity.sh @@ -178,7 +178,7 @@ database_table_from_file() { echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" elif [[ "${table}" == "adlist" ]]; then # Adlist table format - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\",,0,0,0,0,0" >> "${tmpFile}" else # White-, black-, and regexlist table format echo "${rowid},${list_type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${src}\"" >> "${tmpFile}" From 375d4d9bc13b668682ded29efd2387d62d5623c5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jul 2023 21:33:59 +0200 Subject: [PATCH 146/462] Add type as new field of view vw_adlist Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 17712cf7..46f26ba7 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -65,7 +65,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','16'); +INSERT INTO "info" VALUES('version','17'); CREATE TABLE domain_audit ( @@ -158,7 +158,7 @@ CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_i LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; -CREATE VIEW vw_adlist AS SELECT DISTINCT address, id +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type FROM adlist WHERE enabled = 1 ORDER BY id; From 2a03671fb9f15cfe7e6ee378350d7f32d2e04000 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 6 Jul 2023 22:52:28 +0200 Subject: [PATCH 147/462] Reinstall gravity cleanup on error Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 89b4ab84..acf55ae3 100755 --- a/gravity.sh +++ b/gravity.sh @@ -395,7 +395,7 @@ gravity_DownloadBlocklists() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to create new database ${gravityTEMPfile}\\n ${output}" - #gravity_Cleanup "error" + gravity_Cleanup "error" else echo -e "${OVER} ${TICK} ${str}" fi From 5ae0405446103a29c525cba9cdeb12ad390372b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 3 Aug 2023 20:46:01 +0200 Subject: [PATCH 148/462] Ensure pihole-FTL can write custom.list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 4 ++-- automated install/basic-install.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index ff4abf3a..f6e28fec 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -17,9 +17,9 @@ mkdir -pm 0755 /run/pihole /var/log/pihole [ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files -chown pihole:pihole /run/pihole /etc/pihole /var/log/pihole /var/log/pihole/FTL.log /var/log/pihole/pihole.log /etc/pihole/dhcp.leases +chown pihole:pihole /run/pihole /etc/pihole /var/log/pihole /var/log/pihole/FTL.log /var/log/pihole/pihole.log /etc/pihole/dhcp.leases /etc/pihole/custom.list # Ensure that permissions are set so that pihole-FTL can edit the files. We ignore errors as the file may not (yet) exist -chmod -f 0644 /etc/pihole/macvendor.db /etc/pihole/dhcp.leases /var/log/pihole/FTL.log +chmod -f 0644 /etc/pihole/macvendor.db /etc/pihole/dhcp.leases /var/log/pihole/FTL.log /etc/pihole/custom.list chmod -f 0640 /var/log/pihole/pihole.log # Chown database files to the user FTL runs as. We ignore errors as the files may not (yet) exist chown -f pihole:pihole /etc/pihole/pihole-FTL.db /etc/pihole/gravity.db /etc/pihole/macvendor.db diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e3d8ff29..050883cd 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1094,7 +1094,7 @@ installConfigs() { # Install empty custom.list file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then - if ! install -o root -m 644 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then + if ! install -o pihole -g pihole -m 644 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then printf " %b Error: Unable to initialize configuration file %s/custom.list\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}" return 1 fi From a3ea2cd8c311e4c81d7b0654c0893b4bf5cc1e5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 4 Aug 2023 19:32:19 +0200 Subject: [PATCH 149/462] User pihole should be allowed to edit all its files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index f6e28fec..e6a5aeb2 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -17,14 +17,7 @@ mkdir -pm 0755 /run/pihole /var/log/pihole [ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files -chown pihole:pihole /run/pihole /etc/pihole /var/log/pihole /var/log/pihole/FTL.log /var/log/pihole/pihole.log /etc/pihole/dhcp.leases /etc/pihole/custom.list -# Ensure that permissions are set so that pihole-FTL can edit the files. We ignore errors as the file may not (yet) exist -chmod -f 0644 /etc/pihole/macvendor.db /etc/pihole/dhcp.leases /var/log/pihole/FTL.log /etc/pihole/custom.list -chmod -f 0640 /var/log/pihole/pihole.log -# Chown database files to the user FTL runs as. We ignore errors as the files may not (yet) exist -chown -f pihole:pihole /etc/pihole/pihole-FTL.db /etc/pihole/gravity.db /etc/pihole/macvendor.db -# Chmod database file permissions so that the pihole group (web interface) can edit the file. We ignore errors as the files may not (yet) exist -chmod -f 0664 /etc/pihole/pihole-FTL.db +chown -R pihole:pihole /run/pihole /etc/pihole /var/log/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 From 8ef8a275830341140009bdbf229ae8cebad7aede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 4 Aug 2023 19:41:19 +0200 Subject: [PATCH 150/462] Remove webpage.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 100 ------------------------------------ pihole | 41 ++++++++++++--- 2 files changed, 33 insertions(+), 108 deletions(-) delete mode 100755 advanced/Scripts/webpage.sh diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh deleted file mode 100755 index 67cbe766..00000000 --- a/advanced/Scripts/webpage.sh +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090 -# shellcheck disable=SC2154 - - -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Web interface settings -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -# TODO - this entire file might be able to be removed in v6 - -readonly dnsmasqconfig="/etc/dnsmasq.d/01-pihole.conf" -readonly dhcpconfig="/etc/dnsmasq.d/02-pihole-dhcp.conf" -readonly FTLconf="/etc/pihole/pihole-FTL.conf" -# 03 -> wildcards -readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf" -readonly dnscustomfile="/etc/pihole/custom.list" -readonly dnscustomcnamefile="/etc/dnsmasq.d/05-pihole-custom-cname.conf" - -readonly gravityDBfile="/etc/pihole/gravity.db" - - -readonly setupVars="/etc/pihole/setupVars.conf" -readonly PI_HOLE_BIN_DIR="/usr/local/bin" - -# Root of the web server -readonly webroot="/var/www/html" - -# Source utils script -utilsfile="/opt/pihole/utils.sh" -source "${utilsfile}" - -coltable="/opt/pihole/COL_TABLE" -if [[ -f ${coltable} ]]; then - source ${coltable} -fi - -helpFunc() { - echo "Usage: pihole -a [options] -Example: pihole -a -p password -Set options for the API/Web interface - -Options: - -p, password Set API/Web interface password - -h, --help Show this help dialog" - exit 0 -} - -# TODO: We can probably remove the reliance on this function too, just tell people to pihole-FTL --config webserver.api.password "password" -SetWebPassword() { - if (( ${#args[2]} > 0 )) ; then - readonly PASSWORD="${args[2]}" - readonly CONFIRM="${PASSWORD}" - else - # Prevents a bug if the user presses Ctrl+C and it continues to hide the text typed. - # So we reset the terminal via stty if the user does press Ctrl+C - trap '{ echo -e "\nNot changed" ; stty sane ; exit 1; }' INT - read -s -r -p "Enter New Password (Blank for no password): " PASSWORD - echo "" - - if [ "${PASSWORD}" == "" ]; then - setFTLConfigValue "webserver.api.pwhash" "" >/dev/null - echo -e " ${TICK} Password Removed" - exit 0 - fi - - read -s -r -p "Confirm Password: " CONFIRM - echo "" - fi - - if [ "${PASSWORD}" == "${CONFIRM}" ] ; then - # pihole-FTL will automatically hash the password - setFTLConfigValue "webserver.api.password" "${PASSWORD}" >/dev/null - echo -e " ${TICK} New password set" - else - echo -e " ${CROSS} Passwords don't match. Your password has not been changed" - exit 1 - fi -} - -main() { - args=("$@") - - case "${args[1]}" in - "-p" | "password" ) SetWebPassword;; - "-h" | "--help" ) helpFunc;; - * ) helpFunc;; - esac - - shift - - if [[ $# = 0 ]]; then - helpFunc - fi -} diff --git a/pihole b/pihole index 66771b9b..54b20f7d 100755 --- a/pihole +++ b/pihole @@ -30,10 +30,36 @@ if [ -f "${versionsfile}" ]; then source "${versionsfile}" fi -webpageFunc() { - source "${PI_HOLE_SCRIPT_DIR}/webpage.sh" - main "$@" - exit 0 +# TODO: We can probably remove the reliance on this function too, just tell people to pihole-FTL --config webserver.api.password "password" +SetWebPassword() { + if [ -n "$2" ] ; then + readonly PASSWORD="$2" + readonly CONFIRM="${PASSWORD}" + else + # Prevents a bug if the user presses Ctrl+C and it continues to hide the text typed. + # So we reset the terminal via stty if the user does press Ctrl+C + trap '{ echo -e "\nNot changed" ; stty sane ; exit 1; }' INT + read -s -r -p "Enter New Password (Blank for no password): " PASSWORD + echo "" + + if [ "${PASSWORD}" == "" ]; then + setFTLConfigValue "webserver.api.pwhash" "" >/dev/null + echo -e " ${TICK} Password Removed" + exit 0 + fi + + read -s -r -p "Confirm Password: " CONFIRM + echo "" + fi + + if [ "${PASSWORD}" == "${CONFIRM}" ] ; then + # pihole-FTL will automatically hash the password + setFTLConfigValue "webserver.api.password" "${PASSWORD}" >/dev/null + echo -e " ${TICK} New password set" + else + echo -e " ${CROSS} Passwords don't match. Your password has not been changed" + exit 1 + fi } listFunc() { @@ -466,8 +492,7 @@ Debugging Options: Options: - -a, admin Web interface options - Add '-h' for more info on Web Interface usage + setpassword set the password for the web interface -c, chronometer Calculates stats and displays to an LCD Add '-h' for more info on chronometer usage -g, updateGravity Update the list of ad-serving domains @@ -526,7 +551,7 @@ case "${1}" in "restartdns" ) ;; "-g" | "updateGravity" ) need_root=0;; "reloaddns" ) need_root=0;; - "-a" | "admin" ) ;; + "setpassword" ) ;; "checkout" ) ;; "updatechecker" ) ;; "arpflush" ) ;; @@ -581,7 +606,7 @@ case "${1}" in "disable" ) piholeEnable 0 "$2";; "restartdns" ) restartDNS "$2";; "reloaddns" ) restartDNS "reload";; - "-a" | "admin" ) webpageFunc "$@";; + "setpassword" ) SetWebPassword "$@";; "checkout" ) piholeCheckoutFunc "$@";; "updatechecker" ) shift; updateCheckFunc "$@";; "arpflush" ) arpFunc "$@";; From c360743d41620972cfa0b9b10190d010d12bba5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 4 Aug 2023 21:12:40 +0200 Subject: [PATCH 151/462] Re-add file/folder permissions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index e6a5aeb2..ef8e1b88 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -18,6 +18,8 @@ mkdir -pm 0755 /run/pihole /var/log/pihole [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files chown -R pihole:pihole /run/pihole /etc/pihole /var/log/pihole +chmod -R 0640 /var/log/pihole +chmod -R 0660 /etc/pihole /run/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 From 587a2a1c04fa743480d808f74b87da3d8539bc9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 4 Aug 2023 23:52:53 +0200 Subject: [PATCH 152/462] # allow all users to enter der directories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index ef8e1b88..62183db9 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -20,6 +20,8 @@ mkdir -pm 0755 /run/pihole /var/log/pihole chown -R pihole:pihole /run/pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole chmod -R 0660 /etc/pihole /run/pihole +# allow all users to enter der directories +chmod 0755 /etc/pihole /run/pihole /var/log/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 From 7bb0ca59d157f21a50e1e4abcf18734ae2c62347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 5 Aug 2023 14:54:09 +0200 Subject: [PATCH 153/462] Remove traces of /run/pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index 62183db9..a0353f34 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -11,17 +11,17 @@ FTL_PID_FILE="$(getFTLPIDFile)" # Touch files to ensure they exist (create if non-existing, preserve if existing) # shellcheck disable=SC2174 -mkdir -pm 0755 /run/pihole /var/log/pihole +mkdir -pm 0755 /var/log/pihole [ -f "${FTL_PID_FILE}" ] || install -D -m 644 -o pihole -g pihole /dev/null "${FTL_PID_FILE}" [ -f /var/log/pihole/FTL.log ] || install -m 644 -o pihole -g pihole /dev/null /var/log/pihole/FTL.log [ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files -chown -R pihole:pihole /run/pihole /etc/pihole /var/log/pihole +chown -R pihole:pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole chmod -R 0660 /etc/pihole /run/pihole # allow all users to enter der directories -chmod 0755 /etc/pihole /run/pihole /var/log/pihole +chmod 0755 /etc/pihole /var/log/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 From 3c693c1da54b15869d7d612bdf20153234911073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 6 Aug 2023 12:04:55 +0200 Subject: [PATCH 154/462] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: DL6ER Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 4 ++-- automated install/basic-install.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index a0353f34..17900f0b 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -19,8 +19,8 @@ mkdir -pm 0755 /var/log/pihole # Ensure that permissions are set so that pihole-FTL can edit all necessary files chown -R pihole:pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole -chmod -R 0660 /etc/pihole /run/pihole -# allow all users to enter der directories +chmod -R 0660 /etc/pihole +# allow all users to enter the directories chmod 0755 /etc/pihole /var/log/pihole # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 050883cd..bf26631a 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1094,7 +1094,7 @@ installConfigs() { # Install empty custom.list file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then - if ! install -o pihole -g pihole -m 644 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then + if ! install -o pihole -g pihole -m 660 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then printf " %b Error: Unable to initialize configuration file %s/custom.list\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}" return 1 fi From 2227a2c5693633d89fb99fe3a2079d26c9bfe225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 9 Aug 2023 21:25:50 +0200 Subject: [PATCH 155/462] Add API functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 140 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 advanced/Scripts/api.sh diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh new file mode 100644 index 00000000..afd88671 --- /dev/null +++ b/advanced/Scripts/api.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env sh +# shellcheck disable=SC3043 #https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions + +# Pi-hole: A black hole for Internet advertisements +# (c) 2017 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# Script to hold api functions for use in other scripts +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + + +# The basic usage steps are +# 1) Test Availability of the API +# 2) Try to authenticate (read password if needed) +# 3) Get the data from the API endpoint +# 4) Delete the session + + +TestAPIAvailability() { + + # as we are running locally, we can get the port value from FTL directly + PORT="$(pihole-FTL --config webserver.port)" + PORT="${PORT%%,*}" + + availabilityResonse=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${PORT}/api/auth") + + # test if http status code was 200 (OK) or 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then + echo "API not available at: http://localhost:${PORT}/api" + echo "Exiting." + exit 1 + fi +} + +Authenthication() { + # Try to authenticate + LoginAPI + + while [ "${validSession}" = false ] || [ -z "${validSession}" ] ; do + echo "Authentication failed. Please enter your Pi-hole password" + + # secretly read the password + secretRead; printf '\n' + + # Try to authenticate again + LoginAPI + done + + # Loop exited, authentication was successful + echo "Authentication successful." + +} + +LoginAPI() { + sessionResponse="$(curl --silent -X POST "http://localhost:${PORT}/api/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + + if [ -z "${sessionResponse}" ]; then + echo "No response from FTL server. Please check connectivity" + exit 1 + fi + # obtain validity and session ID from session response + validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) + SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) +} + +DeleteSession() { + # if a valid Session exists (no password required or successful authenthication) and + # SID is not null (successful authenthication only), delete the session + if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then + # Try to delete the session. Omit the output, but get the http status code + deleteResponse=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "http://localhost:${PORT}/api/auth" -H "Accept: application/json" -H "sid: ${SID}") + + case "${deleteResponse}" in + "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; + "401") printf "%b" "Logout attempt without a valid session. Unauthorized!\n";; + "410") printf "%b" "Session successfully deleted.\n";; + esac; + fi + +} + +GetFTLData() { + local data + # get the data from querying the API as well as the http status code + data=$(curl -s -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) + echo "${data}" +} + +secretRead() { + + # POSIX compliant function to read user-input and + # mask every character entered by (*) + # + # This is challenging, because in POSIX, `read` does not support + # `-s` option (suppressing the input) or + # `-n` option (reading n chars) + + + # This workaround changes the terminal characteristics to not echo input and later resets this option + # credits https://stackoverflow.com/a/4316765 + # showing asterisk instead of password + # https://stackoverflow.com/a/24600839 + # https://unix.stackexchange.com/a/464963 + + + # Save current terminal settings (needed for later restore after password prompt) + stty_orig=$(stty -g) + + stty -echo # do not echo user input + stty -icanon min 1 time 0 # disable canonical mode https://man7.org/linux/man-pages/man3/termios.3.html + + unset password + unset key + unset charcount + charcount=0 + while key=$(dd ibs=1 count=1 2>/dev/null); do #read one byte of input + if [ "${key}" = "$(printf '\0' | tr -d '\0')" ] ; then + # Enter - accept password + break + fi + if [ "${key}" = "$(printf '\177')" ] ; then + # Backspace + if [ $charcount -gt 0 ] ; then + charcount=$((charcount-1)) + printf '\b \b' + password="${password%?}" + fi + else + # any other character + charcount=$((charcount+1)) + printf '*' + password="$password$key" + fi + done + + # restore original terminal settings + stty "${stty_orig}" +} From bd55b2e5660d31dd5f99e2343ab5c31e871c0075 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Aug 2023 10:25:00 +0000 Subject: [PATCH 156/462] Bump tox from 4.6.4 to 4.7.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.6.4 to 4.7.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.6.4...4.7.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 5a551fa7..c5d002e5 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.6.4 +tox == 4.7.0 From 7747cd22324a28f00a4807787f2d5b6b801653b0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 12 Aug 2023 10:52:02 +0000 Subject: [PATCH 157/462] Bump tox from 4.6.4 to 4.7.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.6.4 to 4.7.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.6.4...4.7.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 5a551fa7..c5d002e5 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.6.4 +tox == 4.7.0 From 8e0a40e71706b70b880a288e0bc93780d4ee54db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 13 Aug 2023 22:00:44 +0200 Subject: [PATCH 158/462] Use api.sh within query.sh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/COL_TABLE | 2 +- advanced/Scripts/query.sh | 325 ++++++++++++------------------------- 2 files changed, 104 insertions(+), 223 deletions(-) mode change 100755 => 100644 advanced/Scripts/query.sh diff --git a/advanced/Scripts/COL_TABLE b/advanced/Scripts/COL_TABLE index 2d2b074b..4344c7b8 100644 --- a/advanced/Scripts/COL_TABLE +++ b/advanced/Scripts/COL_TABLE @@ -1,5 +1,5 @@ # Determine if terminal is capable of showing colors -if ([[ -t 1 ]] && [[ $(tput colors) -ge 8 ]]) || [[ "${WEBCALL}" ]]; then +if ([ -t 1 ] && [ $(tput colors) -ge 8 ]) || [ "${WEBCALL}" ]; then # Bold and underline may not show up on all clients # If something MUST be emphasized, use both COL_BOLD='' diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh old mode 100755 new mode 100644 index bfa21247..8407d3f6 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -1,259 +1,140 @@ -#!/usr/bin/env bash +#!/usr/bin/env sh # shellcheck disable=SC1090 +# Ignore warning about `local` being undefinded in POSIX +# shellcheck disable=SC3043 +# https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions + # Pi-hole: A black hole for Internet advertisements -# (c) 2018 Pi-hole, LLC (https://pi-hole.net) +# (c) 2023 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. # -# Query Domain Lists +# Search Adlists # # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. # Globals -piholeDir="/etc/pihole" -GRAVITYDB="${piholeDir}/gravity.db" -options="$*" -all="" -exact="" -matchType="match" -# Source pihole-FTL from install script -pihole_FTL="${piholeDir}/pihole-FTL.conf" -if [[ -f "${pihole_FTL}" ]]; then - source "${pihole_FTL}" -fi - -# Set this only after sourcing pihole-FTL.conf as the gravity database path may -# have changed -gravityDBfile="${GRAVITYDB}" +PI_HOLE_INSTALL_DIR="/opt/pihole" +max_results="20" +partial="true" +domain="" +# Source color table colfile="/opt/pihole/COL_TABLE" -source "${colfile}" +. "${colfile}" -if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then +# Source api functions +. "${PI_HOLE_INSTALL_DIR}/api.sh" + +Help(){ echo "Usage: pihole -q [option] -Example: 'pihole -q -exact domain.com' +Example: 'pihole -q --exact domain.com' Query the adlists for a specified domain Options: - -exact Search the adlists for exact domain matches - -all Return all query matches within the adlists + --exact Search the adlists for exact domain matches + --all Return all query matches within the adlists -h, --help Show this help dialog" exit 0 -fi - -# Handle valid options -[[ "${options}" == *"-all"* ]] && all=true -if [[ "${options}" == *"-exact"* ]]; then - exact="exact"; matchType="exact ${matchType}" -fi - -# Strip valid options, leaving only the domain and invalid options -# This allows users to place the options before or after the domain -options=$(sed -E 's/ +-(all|exact) ?//g' <<< "${options}") - -# Handle remaining options -# If $options contain non ASCII characters, convert to punycode -case "${options}" in - "" ) str="No domain specified";; - *" "* ) str="Unknown query option specified";; - *[![:ascii:]]* ) rawDomainQuery=$(idn2 "${options}");; - * ) rawDomainQuery="${options}";; -esac - -# convert the domain to lowercase -domainQuery=$(echo "${rawDomainQuery}" | tr '[:upper:]' '[:lower:]') - -if [[ -n "${str:-}" ]]; then - echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information." - exit 1 -fi - -# Scan a domain again a list of RegEX -scanRegExList(){ - local domain="${1}" list="${2}" - - for entry in ${list}; do - if [[ "${domain}" =~ ${entry} ]]; then - printf "%b\n" "${entry}"; - fi - done - } -scanDatabaseTable() { - local domain table list_type querystr result extra abpquerystr abpfound abpentry searchstr - domain="$(printf "%q" "${1}")" - table="${2}" - list_type="${3:-}" - # As underscores are legitimate parts of domains, we escape them when using the LIKE operator. - # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this - # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched - # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. - if [[ "${table}" == "gravity" ]]; then +GenerateOutput(){ + local data gravity_data lists_data num_gravity num_lists search_type_str + local gravity_data_csv lists_data_csv line current_domain + data="${1}" - # Are there ABP entries on gravity? - # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set - abpquerystr="SELECT EXISTS (SELECT 1 FROM info WHERE property='abp_domains' and value='1')" - abpfound="$(pihole-FTL sqlite3 "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null + # construct a new json for the list results where each object contains the domain and the related type + lists_data=$(echo "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') - # Create search string for ABP entries only if needed - if [ "${abpfound}" -eq 1 ]; then - abpentry="${domain}" + # construct a new json for the gravity results where each object contains the adlist URL and the related domains + gravity_data=$(echo "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') - searchstr="'||${abpentry}^'" + # number of objects in each json + num_gravity=$(echo "${gravity_data}" | jq length ) + num_lists=$(echo "${lists_data}" | jq length ) - # While a dot is found ... - while [ "${abpentry}" != "${abpentry/./}" ] - do - # ... remove text before the dot (including the dot) and append the result to $searchstr - abpentry=$(echo "${abpentry}" | cut -f 2- -d '.') - searchstr="$searchstr, '||${abpentry}^'" + if [ "${partial}" = true ]; then + search_type_str="partially" + else + search_type_str="exactly" + fi + + # Results from allow/deny list + printf "%s\n\n" "Found ${num_lists} domains ${search_type_str} matching '${COL_BLUE}${domain}${COL_NC}'." + if [ "${num_lists}" -gt 0 ]; then + # Convert the data to a csv, each line is a "domain,type" string + # not using jq's @csv here as it quotes each value individually + lists_data_csv=$(echo "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")' ) + + # Generate output for each csv line, separating line in a domain and type substring at the ',' + echo "${lists_data_csv}" | while read -r line; do + printf "%s\n\n" " - ${COL_GREEN}${line%,*}${COL_NC} (type: exact ${line#*,} domain)" + done + fi + + # Results from gravity + printf "%s\n\n" "Found ${num_gravity} adlists ${search_type_str} matching '${COL_BLUE}${domain}${COL_NC}'." + if [ "${num_gravity}" -gt 0 ]; then + # Convert the data to a csv, each line is a "URL,domain,domain,...." string + # not using jq's @csv here as it quotes each value individually + gravity_data_csv=$(echo "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")' ) + + # Generate line-by-line output for each csv line + echo "${gravity_data_csv}" | while read -r line; do + + # print adlist URL + printf "%s\n\n" " - ${COL_BLUE}${line%%,*}${COL_NC}" + + # cut off URL, leaving "domain,domain,...." + line=${line#*,} + # print each domain and remove it from the string until nothing is left + while [ ${#line} -gt 0 ]; do + current_domain=${line%%,*} + printf ' - %s\n' "${COL_GREEN}${current_domain}${COL_NC}" + # we need to remove the current_domain and the comma in two steps because + # the last domain won't have a trailing comma and the while loop wouldn't exit + line=${line#"${current_domain}"} + line=${line#,} done - - # The final search string will look like: - # "domain IN ('||sub2.sub1.domain.com^', '||sub1.domain.com^', '||domain.com^', '||com^') OR" - searchstr="domain IN (${searchstr}) OR " - fi - - case "${exact}" in - "exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";; - * ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE ${searchstr} domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; - esac - else - case "${exact}" in - "exact" ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain = '${domain}'";; - * ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; - esac - fi - - # Send prepared query to gravity database - result="$(pihole-FTL sqlite3 -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null - if [[ -z "${result}" ]]; then - # Return early when there are no matches in this table - return - fi - - if [[ "${table}" == "gravity" ]]; then - echo "${result}" - return - fi - - # Mark domain as having been white-/blacklist matched (global variable) - wbMatch=true - - # Print table name - echo " ${matchType^} found in ${COL_BOLD}exact ${table}${COL_NC}" - - # Loop over results and print them - mapfile -t results <<< "${result}" - for result in "${results[@]}"; do - domain="${result/,*}" - if [[ "${result#*,}" == "0" ]]; then - extra=" (disabled)" - else - extra="" - fi - echo " ${domain}${extra}" - done -} - -scanRegexDatabaseTable() { - local domain list list_type - domain="${1}" - list="${2}" - list_type="${3:-}" - - # Query all regex from the corresponding database tables - mapfile -t regexList < <(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${list_type}" 2> /dev/null) - - # If we have regexps to process - if [[ "${#regexList[@]}" -ne 0 ]]; then - # Split regexps over a new line - str_regexList=$(printf '%s\n' "${regexList[@]}") - # Check domain against regexps - mapfile -t regexMatches < <(scanRegExList "${domain}" "${str_regexList}") - # If there were regex matches - if [[ "${#regexMatches[@]}" -ne 0 ]]; then - # Split matching regexps over a new line - str_regexMatches=$(printf '%s\n' "${regexMatches[@]}") - # Form a "matched" message - str_message="${matchType^} found in ${COL_BOLD}regex ${list}${COL_NC}" - # Form a "results" message - str_result="${COL_BOLD}${str_regexMatches}${COL_NC}" - # If we are displaying more than just the source of the block - # Set the wildcard match flag - wcMatch=true - # Echo the "matched" message, indented by one space - echo " ${str_message}" - # Echo the "results" message, each line indented by three spaces - # shellcheck disable=SC2001 - echo "${str_result}" | sed 's/^/ /' - fi + printf "\n\n" + done fi } -# Scan Whitelist and Blacklist -scanDatabaseTable "${domainQuery}" "whitelist" "0" -scanDatabaseTable "${domainQuery}" "blacklist" "1" +Main(){ + local data -# Scan Regex table -scanRegexDatabaseTable "${domainQuery}" "whitelist" "2" -scanRegexDatabaseTable "${domainQuery}" "blacklist" "3" - -# Query block lists -mapfile -t results <<< "$(scanDatabaseTable "${domainQuery}" "gravity")" - -# Handle notices -if [[ -z "${wbMatch:-}" ]] && [[ -z "${wcMatch:-}" ]] && [[ -z "${results[*]}" ]]; then - echo -e " ${INFO} No ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} within the adlists" - exit 0 -elif [[ -z "${results[*]}" ]]; then - # Result found in WL/BL/Wildcards - exit 0 -elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 100 ]]; then - echo -e " ${INFO} Over 100 ${exact/t/t }results found for ${COL_BOLD}${domainQuery}${COL_NC} - This can be overridden using the -all option" - exit 0 -fi - -# Print "Exact matches for" title -if [[ -n "${exact}" ]]; then - plural=""; [[ "${#results[*]}" -gt 1 ]] && plural="es" - echo " ${matchType^}${plural} for ${COL_BOLD}${domainQuery}${COL_NC} found in:" -fi - -for result in "${results[@]}"; do - match="${result/,*/}" - extra="${result#*,}" - adlistAddress="${extra/,*/}" - extra="${extra#*,}" - if [[ "${extra}" == "0" ]]; then - extra=" (disabled)" + if [ -z "${domain}" ]; then + echo "No domain specified"; exit 1 else - extra="" + # convert domain to punycode + domain=$(idn2 "${domain}") + + # convert the domain to lowercase + domain=$(echo "${domain}" | tr '[:upper:]' '[:lower:]') fi - if [[ -n "${exact}" ]]; then - echo " - ${adlistAddress}${extra}" - else - if [[ ! "${adlistAddress}" == "${adlistAddress_prev:-}" ]]; then - count="" - echo " ${matchType^} found in ${COL_BOLD}${adlistAddress}${COL_NC}:" - adlistAddress_prev="${adlistAddress}" - fi - : $((count++)) + # Test if the authentication endpoint is available + TestAPIAvailability + # Authenticate with the FTL server + Authenthication - # Print matching domain if $max_count has not been reached - [[ -z "${all}" ]] && max_count="50" - if [[ -z "${all}" ]] && [[ "${count}" -ge "${max_count}" ]]; then - [[ "${count}" -gt "${max_count}" ]] && continue - echo " ${COL_GRAY}Over ${count} results found, skipping rest of file${COL_NC}" - else - echo " ${match}${extra}" - fi - fi + data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + GenerateOutput "${data}" + DeleteSession +} + +# Process all options (if present) +while [ "$#" -gt 0 ]; do + case "$1" in + "-h" | "--help" ) Help;; + "--exact" ) partial="false";; + "--all" ) max_results=10000;; # hard-coded FTL limit + * ) domain=$1;; + esac + shift done -exit 0 +Main "${domain}" From ee4eb8db201a375afe3fdf9f7af1d52ce2a630cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 13 Aug 2023 22:26:44 +0200 Subject: [PATCH 159/462] chmod +x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 0 advanced/Scripts/query.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 advanced/Scripts/api.sh mode change 100644 => 100755 advanced/Scripts/query.sh diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh old mode 100644 new mode 100755 diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh old mode 100644 new mode 100755 From 43882693a5d72efdd2fb91b593a6b50dc9041b92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 19 Aug 2023 10:19:26 +0000 Subject: [PATCH 160/462] Bump tox from 4.7.0 to 4.9.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.7.0 to 4.9.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.7.0...4.9.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index c5d002e5..0210aa4e 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.7.0 +tox == 4.9.0 From 272ca8c55d0aa2f245ac1a466fd2beabca43f026 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 19 Aug 2023 10:48:18 +0000 Subject: [PATCH 161/462] Bump tox from 4.7.0 to 4.9.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.7.0 to 4.9.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.7.0...4.9.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index c5d002e5..0210aa4e 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 8.1.0 -tox == 4.7.0 +tox == 4.9.0 From 1c6919cf07a36c280f02dddd840e036a209747c6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 10:08:08 +0000 Subject: [PATCH 162/462] Bump pytest-testinfra from 8.1.0 to 9.0.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 8.1.0 to 9.0.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/8.1.0...9.0.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0210aa4e..b1b36de1 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 -pytest-testinfra == 8.1.0 +pytest-testinfra == 9.0.0 tox == 4.9.0 From 924f8b88448996c8b2f11be7c11aefbb6d928e79 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 10:20:12 +0000 Subject: [PATCH 163/462] Bump actions/checkout from 3.5.3 to 3.6.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.5.3 to 3.6.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.5.3...v3.6.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7fd0e4e4..1194ba0f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 43d5ca96..13b05df2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 5435b7db..60f38cf6 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8546e02b..01792fa4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Check scripts in repository are executable run: | @@ -65,7 +65,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From 1afc96c05550753a84ad0defe1a9bcf8c95c5970 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 10:46:28 +0000 Subject: [PATCH 164/462] Bump pytest-testinfra from 8.1.0 to 9.0.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 8.1.0 to 9.0.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/8.1.0...9.0.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0210aa4e..b1b36de1 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 -pytest-testinfra == 8.1.0 +pytest-testinfra == 9.0.0 tox == 4.9.0 From e65b171aea04d8f40dceaf5fa4706a2f0b793f14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 11:24:09 +0000 Subject: [PATCH 165/462] Bump tox from 4.9.0 to 4.10.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.9.0 to 4.10.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.9.0...4.10.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index b1b36de1..90255f7f 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.9.0 +tox == 4.10.0 From 53e8127781645d5af39644d95c6ffd894bb8d8cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 26 Aug 2023 11:33:09 +0000 Subject: [PATCH 166/462] Bump actions/checkout from 3.5.3 to 3.6.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.5.3 to 3.6.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.5.3...v3.6.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 7fd0e4e4..1194ba0f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 43d5ca96..13b05df2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 5435b7db..60f38cf6 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 09462eb4..6c76e9fd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.5.3 + uses: actions/checkout@v3.6.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From 9b68fa0b27c5b44a4f7baf211c051a6164c2c6df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 27 Aug 2023 12:28:54 +0000 Subject: [PATCH 167/462] Bump tox from 4.9.0 to 4.10.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.9.0 to 4.10.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.9.0...4.10.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index b1b36de1..90255f7f 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.9.0 +tox == 4.10.0 From eca84e0986c7373ebfe057447fa244f3ecf3dba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 21 Aug 2023 21:28:17 +0200 Subject: [PATCH 168/462] Remove user agent when downloading adlists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/gravity.sh b/gravity.sh index ed402a34..248aa2da 100755 --- a/gravity.sh +++ b/gravity.sh @@ -382,7 +382,7 @@ gravity_DownloadBlocklists() { unset sources fi - local url domain agent str target compression + local url domain str target compression echo "" # Prepare new gravity database @@ -438,9 +438,6 @@ gravity_DownloadBlocklists() { saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}" activeDomains[$i]="${saveLocation}" - # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-) - agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" - echo -e " ${INFO} Target: ${url}" local regex check_url # Check for characters NOT allowed in URLs @@ -453,7 +450,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" + gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" fi echo "" done @@ -485,7 +482,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" + local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM @@ -545,7 +542,7 @@ gravity_DownloadBlocklistFromUrl() { fi # shellcheck disable=SC2086 - httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) + httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) case $url in # Did we "download" a local file? From c33963b25000272d023ff00f1c910bab3c891429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 4 Jun 2023 22:51:10 +0200 Subject: [PATCH 169/462] Check for valid OS via IPv4 and IPv6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 17 ++++++++++++++-- automated install/basic-install.sh | 31 ++++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c3bc81b0..622ebd7a 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -339,7 +339,7 @@ os_check() { detected_os=$(grep "\bID\b" /etc/os-release | cut -d '=' -f2 | tr -d '"') detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') - cmdResult="$(dig +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" + cmdResult="$(dig -4 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" #Get the return code of the previous command (last line) digReturnCode="${cmdResult##*$'\n'}" @@ -349,7 +349,20 @@ os_check() { if [ "${digReturnCode}" -ne 0 ]; then log_write "${INFO} Distro: ${detected_os^}" log_write "${INFO} Version: ${detected_version}" - log_write "${CROSS} dig return code: ${COL_RED}${digReturnCode}${COL_NC}" + log_write "${CROSS} dig IPv4 return code: ${COL_RED}${digReturnCode}${COL_NC}" + log_write "${CROSS} dig response: ${response}" + log_write "${INFO} Retrying via IPv6" + + cmdResult="$(dig -6 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" + #Get the return code of the previous command (last line) + digReturnCode="${cmdResult##*$'\n'}" + + # Extract dig response + response="${cmdResult%%$'\n'*}" + fi + # If also no success via IPv6 + if [ "${digReturnCode}" -ne 0 ]; then + log_write "${CROSS} dig IPv6 return code: ${COL_RED}${digReturnCode}${COL_NC}" log_write "${CROSS} dig response: ${response}" log_write "${CROSS} Error: ${COL_RED}dig command failed - Unable to check OS${COL_NC}" else diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e3d8ff29..990b8789 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -176,7 +176,8 @@ os_check() { detected_os=$(grep '^ID=' /etc/os-release | cut -d '=' -f2 | tr -d '"') detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') - cmdResult="$(dig +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" + # Test via IPv4 + cmdResult="$(dig -4 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" # Gets the return code of the previous command (last line) digReturnCode="${cmdResult##*$'\n'}" @@ -188,8 +189,34 @@ os_check() { # If the value of ${response} is a single 0, then this is the return code, not an actual response. if [ "${response}" == 0 ]; then valid_response=false + else + valid_response=true fi + fi + # Try again via IPv6 + if [ "$valid_response" = false ]; then + unset valid_response + + cmdResult="$(dig -6 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" + # Gets the return code of the previous command (last line) + digReturnCode="${cmdResult##*$'\n'}" + + if [ ! "${digReturnCode}" == "0" ]; then + valid_response=false + else + # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid + response="${cmdResult%%$'\n'*}" + # If the value of ${response} is a single 0, then this is the return code, not an actual response. + if [ "${response}" == 0 ]; then + valid_response=false + else + valid_response=true + fi + fi + fi + + if [ "$valid_response" = true ]; then IFS=" " read -r -a supportedOS < <(echo "${response}" | tr -d '"') for distro_and_versions in "${supportedOS[@]}" do @@ -212,7 +239,7 @@ os_check() { done fi - if [ "$valid_os" = true ] && [ "$valid_version" = true ] && [ ! "$valid_response" = false ]; then + if [ "$valid_os" = true ] && [ "$valid_version" = true ] && [ "$valid_response" = true ]; then display_warning=false fi From ff2c2290c7390ef73a34b1c1255af12efe476b61 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Sep 2023 10:27:10 +0000 Subject: [PATCH 170/462] Bump tox from 4.10.0 to 4.11.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.10.0 to 4.11.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.10.0...4.11.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 90255f7f..7f4d7e09 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.10.0 +tox == 4.11.1 From 9084b170cb162fdd9f615fe6b66d8e1dfa95a5b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Sep 2023 10:35:16 +0000 Subject: [PATCH 171/462] Bump tox from 4.10.0 to 4.11.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.10.0 to 4.11.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.10.0...4.11.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 90255f7f..7f4d7e09 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.10.0 +tox == 4.11.1 From cdc1c1ace10e4425dde0870ffb57a3c945910057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 21 Aug 2023 21:28:17 +0200 Subject: [PATCH 172/462] Remove user agent when downloading adlists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/gravity.sh b/gravity.sh index 2bd21d01..db199090 100755 --- a/gravity.sh +++ b/gravity.sh @@ -401,7 +401,7 @@ gravity_DownloadBlocklists() { unset sources fi - local url domain agent str target compression + local url domain str target compression echo "" # Prepare new gravity database @@ -457,9 +457,6 @@ gravity_DownloadBlocklists() { saveLocation="${piholeDir}/list.${id}.${domain}.${domainsExtension}" activeDomains[$i]="${saveLocation}" - # Default user-agent (for Cloudflare's Browser Integrity Check: https://support.cloudflare.com/hc/en-us/articles/200170086-What-does-the-Browser-Integrity-Check-do-) - agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" - echo -e " ${INFO} Target: ${url}" local regex check_url # Check for characters NOT allowed in URLs @@ -472,7 +469,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${agent}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" + gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" fi echo "" done @@ -504,7 +501,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" agent="${2}" adlistID="${3}" saveLocation="${4}" target="${5}" compression="${6}" + local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM @@ -564,7 +561,7 @@ gravity_DownloadBlocklistFromUrl() { fi # shellcheck disable=SC2086 - httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" -A "${agent}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) + httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) case $url in # Did we "download" a local file? From 843f57399c289982cbcd1629030e685b13f01f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 7 Sep 2023 22:27:49 +0200 Subject: [PATCH 173/462] Ignore ABP style entries in debug log dig test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 6f747855..ecb4fc19 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -861,11 +861,15 @@ dig_at() { local record_type="A" fi - # Find a random blocked url that has not been whitelisted. + # Find a random blocked url that has not been whitelisted and is not ABP style. # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1") + random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") + # Falback if no non-ABP style domains were found + if [ -z "${random_url}" ]; then + random_url="flurry.com" + fi # Next we need to check if Pi-hole can resolve a domain when the query is sent to it's IP address # This better emulates how clients will interact with Pi-hole as opposed to above where Pi-hole is From 52268f01550acf609ddac0d7726cfb3fe03e4cc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 7 Sep 2023 22:27:49 +0200 Subject: [PATCH 174/462] Ignore ABP style entries in debug log dig test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c3bc81b0..6ecb49b4 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -809,11 +809,15 @@ dig_at() { local record_type="A" fi - # Find a random blocked url that has not been whitelisted. + # Find a random blocked url that has not been whitelisted and is not ABP style. # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1") + random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") + # Falback if no non-ABP style domains were found + if [ -z "${random_url}" ]; then + random_url="flurry.com" + fi # Next we need to check if Pi-hole can resolve a domain when the query is sent to it's IP address # This better emulates how clients will interact with Pi-hole as opposed to above where Pi-hole is From a229a623bbb282c7aa72e4485f4820f273c7fd5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 10:05:32 +0000 Subject: [PATCH 175/462] Bump tox from 4.11.1 to 4.11.3 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.1 to 4.11.3. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.1...4.11.3) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 7f4d7e09..139be0fc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.11.1 +tox == 4.11.3 From 1f241a3d451ea6bf36eb5feb07255b8bdf91d82f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 10:08:40 +0000 Subject: [PATCH 176/462] Bump actions/checkout from 3.6.0 to 4.0.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.6.0 to 4.0.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.6.0...v4.0.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1194ba0f..570fabdd 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 13b05df2..8d09e5e0 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 60f38cf6..184319e0 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 01792fa4..745a250c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Check scripts in repository are executable run: | @@ -65,7 +65,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From 16ea50ad552de3cb70ffc6637614b1f3ef4878e0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 10:17:15 +0000 Subject: [PATCH 177/462] Bump actions/checkout from 3.6.0 to 4.0.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 3.6.0 to 4.0.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3.6.0...v4.0.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1194ba0f..570fabdd 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 13b05df2..8d09e5e0 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 60f38cf6..184319e0 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6c76e9fd..bf027210 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v3.6.0 + uses: actions/checkout@v4.0.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From 047eac6d9c400556c107bd6fce0d28ff3f92ac4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 10:58:28 +0000 Subject: [PATCH 178/462] Bump tox from 4.11.1 to 4.11.3 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.1 to 4.11.3. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.1...4.11.3) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 7f4d7e09..139be0fc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.0 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 -tox == 4.11.1 +tox == 4.11.3 From 55f72ac92584342cf98870c9735554b0aa8df819 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 10:58:32 +0000 Subject: [PATCH 179/462] Bump pytest from 7.4.0 to 7.4.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.0 to 7.4.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.0...7.4.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 7f4d7e09..9279f97d 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.0 +pytest == 7.4.2 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 tox == 4.11.1 From dc73ace7c46c72d5692945026ae0fea0e45f5756 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Sep 2023 15:07:06 +0000 Subject: [PATCH 180/462] Bump pytest from 7.4.0 to 7.4.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.0 to 7.4.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.0...7.4.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 139be0fc..27417754 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.0 +pytest == 7.4.2 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 tox == 4.11.3 From df92b8ac14988ddeece92b733dec7e8b9fb84d3a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 12 Sep 2023 20:43:34 +0200 Subject: [PATCH 181/462] Add missing creation of view vw_antigravity as well as schema change to vw_adlist Signed-off-by: DL6ER --- .../database_migration/gravity/15_to_16.sql | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity/15_to_16.sql b/advanced/Scripts/database_migration/gravity/15_to_16.sql index c6159f40..7c3b8197 100644 --- a/advanced/Scripts/database_migration/gravity/15_to_16.sql +++ b/advanced/Scripts/database_migration/gravity/15_to_16.sql @@ -6,6 +6,20 @@ BEGIN TRANSACTION; ALTER TABLE adlist ADD COLUMN abp_entries INTEGER NOT NULL DEFAULT 0; +CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + +DROP VIEW vw_adlist; + +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type + FROM adlist + WHERE enabled = 1 + ORDER BY id; + UPDATE info SET value = 16 WHERE property = 'version'; COMMIT; From ea23c8364dd75190b1a8765e8a302877063b9b99 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 16 Sep 2023 10:04:29 +0200 Subject: [PATCH 182/462] Move antigravity-related changed to gravits database migration step 16->17 Signed-off-by: DL6ER --- .../database_migration/gravity/15_to_16.sql | 14 -------------- .../database_migration/gravity/16_to_17.sql | 14 ++++++++++++++ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity/15_to_16.sql b/advanced/Scripts/database_migration/gravity/15_to_16.sql index 7c3b8197..c6159f40 100644 --- a/advanced/Scripts/database_migration/gravity/15_to_16.sql +++ b/advanced/Scripts/database_migration/gravity/15_to_16.sql @@ -6,20 +6,6 @@ BEGIN TRANSACTION; ALTER TABLE adlist ADD COLUMN abp_entries INTEGER NOT NULL DEFAULT 0; -CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id - FROM antigravity - LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id - LEFT JOIN adlist ON adlist.id = antigravity.adlist_id - LEFT JOIN "group" ON "group".id = adlist_by_group.group_id - WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; - -DROP VIEW vw_adlist; - -CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type - FROM adlist - WHERE enabled = 1 - ORDER BY id; - UPDATE info SET value = 16 WHERE property = 'version'; COMMIT; diff --git a/advanced/Scripts/database_migration/gravity/16_to_17.sql b/advanced/Scripts/database_migration/gravity/16_to_17.sql index c7b9049b..23532e3a 100644 --- a/advanced/Scripts/database_migration/gravity/16_to_17.sql +++ b/advanced/Scripts/database_migration/gravity/16_to_17.sql @@ -8,6 +8,20 @@ ALTER TABLE adlist ADD COLUMN type INTEGER NOT NULL DEFAULT 0; UPDATE adlist SET type = 0; +CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + +DROP VIEW vw_adlist; + +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id, type + FROM adlist + WHERE enabled = 1 + ORDER BY id; + UPDATE info SET value = 17 WHERE property = 'version'; COMMIT; From c3d1f366ec2b302b58488b45803a7707347e0c21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Sep 2023 10:39:24 +0000 Subject: [PATCH 183/462] Bump actions/checkout from 4.0.0 to 4.1.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.0.0 to 4.1.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.0.0...v4.1.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 570fabdd..4685aa2c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index b783ff25..14c55d10 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 184319e0..a1025629 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 745a250c..ae03db07 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Check scripts in repository are executable run: | @@ -65,7 +65,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From ec9d84692f55c3069ba895631437a050c7199086 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Sep 2023 10:45:09 +0000 Subject: [PATCH 184/462] Bump actions/checkout from 4.0.0 to 4.1.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.0.0 to 4.1.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.0.0...v4.1.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 570fabdd..4685aa2c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 8d09e5e0..0ff0a24a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 184319e0..a1025629 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf027210..054c09ac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.0.0 + uses: actions/checkout@v4.1.0 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.0 From aba41b45b0e7ff67ed88cb6f706bdddb3bbadeb9 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 4 Oct 2023 16:20:38 +0100 Subject: [PATCH 185/462] Some verbiage change to outputs Signed-off-by: Adam Warner --- gravity.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index eced477d..9133d33d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -675,10 +675,10 @@ gravity_ShowCount() { # Here we use the table "gravity" instead of the view "vw_gravity" for speed. # It's safe to replace it here, because right after a gravity run both will show the exactly same number of domains. gravity_Table_Count "gravity" "gravity domains" "" - gravity_Table_Count "vw_blacklist" "exact blacklisted domains" - gravity_Table_Count "vw_regex_blacklist" "regex blacklist filters" - gravity_Table_Count "vw_whitelist" "exact whitelisted domains" - gravity_Table_Count "vw_regex_whitelist" "regex whitelist filters" + gravity_Table_Count "vw_blacklist" "exact denied domains" + gravity_Table_Count "vw_regex_blacklist" "regex denied filters" + gravity_Table_Count "vw_whitelist" "exact allowed domains" + gravity_Table_Count "vw_regex_whitelist" "regex allowed filters" } # Create "localhost" entries into hosts format From 885b626a68099bd44eb952307857438513a45cc9 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 4 Oct 2023 16:22:01 +0100 Subject: [PATCH 186/462] Some unrelated spelling mistakes that spellcheck is grumbling about Signed-off-by: Adam Warner --- advanced/Scripts/piholeDebug.sh | 6 +++--- automated install/basic-install.sh | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 6ecb49b4..0e3bbf3d 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -397,7 +397,7 @@ os_check() { } diagnose_operating_system() { - # error message in a variable so we can easily modify it later (or re-use it) + # error message in a variable so we can easily modify it later (or reuse it) local error_msg="Distribution unknown -- most likely you are on an unsupported platform and may run into issues." # Display the current test that is running echo_current_diagnostic "Operating system" @@ -814,7 +814,7 @@ dig_at() { # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") - # Falback if no non-ABP style domains were found + # Fallback if no non-ABP style domains were found if [ -z "${random_url}" ]; then random_url="flurry.com" fi @@ -1451,7 +1451,7 @@ upload_to_tricorder() { # If no token was generated else # Show an error and some help instructions - # Skip this if being called from web interface and autmatic mode was not chosen (users opt-out to upload) + # Skip this if being called from web interface and automatic mode was not chosen (users opt-out to upload) if [[ "${WEBCALL}" ]] && [[ ! "${AUTOMATED}" ]]; then : else diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index bf26631a..4e4bdfc7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2203,7 +2203,7 @@ main() { # Check for and disable systemd-resolved-DNSStubListener before reloading resolved # DNSStubListener needs to remain in place for installer to download needed files, # so this change needs to be made after installation is complete, - # but before starting or resarting the ftl service + # but before starting or restarting the ftl service disable_resolved_stublistener printf " %b Restarting services...\\n" "${INFO}" From edf0060acf97e7072575599bcdbeab45541b568e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 6 Oct 2023 22:26:08 +0200 Subject: [PATCH 187/462] Fix spellcheck errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .codespellignore | 1 + advanced/Scripts/piholeDebug.sh | 4 ++-- automated install/basic-install.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.codespellignore b/.codespellignore index 501a3d67..6d05d295 100644 --- a/.codespellignore +++ b/.codespellignore @@ -1,3 +1,4 @@ doubleclick wan nwe +re-use diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index ecb4fc19..47592686 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -866,7 +866,7 @@ dig_at() { # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") - # Falback if no non-ABP style domains were found + # Fallback if no non-ABP style domains were found if [ -z "${random_url}" ]; then random_url="flurry.com" fi @@ -1503,7 +1503,7 @@ upload_to_tricorder() { # If no token was generated else # Show an error and some help instructions - # Skip this if being called from web interface and autmatic mode was not chosen (users opt-out to upload) + # Skip this if being called from web interface and automatic mode was not chosen (users opt-out to upload) if [[ "${WEBCALL}" ]] && [[ ! "${AUTOMATED}" ]]; then : else diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1f3002e7..e87edd71 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2685,7 +2685,7 @@ main() { # Check for and disable systemd-resolved-DNSStubListener before reloading resolved # DNSStubListener needs to remain in place for installer to download needed files, # so this change needs to be made after installation is complete, - # but before starting or resarting the dnsmasq or ftl services + # but before starting or restarting the dnsmasq or ftl services disable_resolved_stublistener # If the Web server was installed, From afa688e3a0e565e797804051bd74888712484ec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 7 Oct 2023 00:21:06 +0200 Subject: [PATCH 188/462] Yu vs uh (https://www.theguardian.com/guardian-observer-style-guide-r#:~:text=re/re%2D,re%2Dsign/resign) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .codespellignore | 1 - advanced/Scripts/piholeDebug.sh | 2 +- test/test_any_automated_install.py | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.codespellignore b/.codespellignore index 6d05d295..501a3d67 100644 --- a/.codespellignore +++ b/.codespellignore @@ -1,4 +1,3 @@ doubleclick wan nwe -re-use diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 47592686..1ca52f45 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -449,7 +449,7 @@ os_check() { } diagnose_operating_system() { - # error message in a variable so we can easily modify it later (or re-use it) + # error message in a variable so we can easily modify it later (or reuse it) local error_msg="Distribution unknown -- most likely you are on an unsupported platform and may run into issues." # Display the current test that is running echo_current_diagnostic "Operating system" diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 36a4f0d2..840d1df7 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -70,7 +70,7 @@ def test_setupVars_are_sourced_to_global_scope(host): def test_setupVars_saved_to_file(host): """ - confirm saved settings are written to a file for future updates to re-use + confirm saved settings are written to a file for future updates to reuse """ # dedent works better with this and padding matching script below set_setup_vars = "\n" From f52b2b98630dc0f606fed94d74abee6dc2b22582 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Oct 2023 10:23:00 +0000 Subject: [PATCH 189/462] Bump actions/setup-python from 4.7.0 to 4.7.1 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.0 to 4.7.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.0...v4.7.1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ae03db07..623590da 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: uses: actions/checkout@v4.1.0 - name: Set up Python 3.10 - uses: actions/setup-python@v4.7.0 + uses: actions/setup-python@v4.7.1 with: python-version: "3.10" From 70547755d661add4df58a39b18db01543cac2141 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 7 Oct 2023 10:39:32 +0000 Subject: [PATCH 190/462] Bump actions/setup-python from 4.7.0 to 4.7.1 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.0 to 4.7.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.0...v4.7.1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 054c09ac..9f32302e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: uses: actions/checkout@v4.1.0 - name: Set up Python 3.10 - uses: actions/setup-python@v4.7.0 + uses: actions/setup-python@v4.7.1 with: python-version: "3.10" From 044e856e6bd84a128b46930fb8a1a3489cf16ad1 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 8 Oct 2023 13:23:44 +0100 Subject: [PATCH 191/462] Disable checkout function for (official) docker containers Signed-off-by: Adam Warner --- pihole | 38 +++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/pihole b/pihole index 54b20f7d..5281c715 100755 --- a/pihole +++ b/pihole @@ -413,26 +413,30 @@ tailFunc() { } piholeCheckoutFunc() { - if [[ "$2" == "-h" ]] || [[ "$2" == "--help" ]]; then - echo "Usage: pihole checkout [repo] [branch] -Example: 'pihole checkout master' or 'pihole checkout core dev' -Switch Pi-hole subsystems to a different GitHub branch + if [ -n "${DOCKER_VERSION}" ]; then + unsupportedFunc + else + if [[ "$2" == "-h" ]] || [[ "$2" == "--help" ]]; then + echo "Usage: pihole checkout [repo] [branch] + Example: 'pihole checkout master' or 'pihole checkout core dev' + Switch Pi-hole subsystems to a different GitHub branch -Repositories: - core [branch] Change the branch of Pi-hole's core subsystem - web [branch] Change the branch of Web Interface subsystem - ftl [branch] Change the branch of Pi-hole's FTL subsystem + Repositories: + core [branch] Change the branch of Pi-hole's core subsystem + web [branch] Change the branch of Web Interface subsystem + ftl [branch] Change the branch of Pi-hole's FTL subsystem -Branches: - master Update subsystems to the latest stable release - dev Update subsystems to the latest development release - branchname Update subsystems to the specified branchname" - exit 0 + Branches: + master Update subsystems to the latest stable release + dev Update subsystems to the latest development release + branchname Update subsystems to the specified branchname" + exit 0 + fi + + source "${PI_HOLE_SCRIPT_DIR}"/piholeCheckout.sh + shift + checkout "$@" fi - - source "${PI_HOLE_SCRIPT_DIR}"/piholeCheckout.sh - shift - checkout "$@" } tricorderFunc() { From 7886dc017266a63041bbbf2f3c751452a6661d9e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 8 Oct 2023 14:09:47 +0100 Subject: [PATCH 192/462] adminlte->web Signed-off-by: Adam Warner --- README.md | 2 +- advanced/Scripts/chronometer.sh | 2 +- advanced/Scripts/update.sh | 2 +- advanced/Scripts/updatecheck.sh | 4 ++-- advanced/Scripts/version.sh | 16 ++++++++-------- automated install/basic-install.sh | 4 ++-- manpages/pihole.8 | 4 ++-- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index adfd3450..eb50030b 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ You can read our [Core Feature Breakdown](https://docs.pi-hole.net/core/pihole-c ### The Web Interface Dashboard -This [optional dashboard](https://github.com/pi-hole/AdminLTE) allows you to view stats, change settings, and configure your Pi-hole. It's the power of the Command Line Interface, with none of the learning curve! +This [optional dashboard](https://github.com/pi-hole/web) allows you to view stats, change settings, and configure your Pi-hole. It's the power of the Command Line Interface, with none of the learning curve! Some notable features include: diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index 49de6efd..fc728e17 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -225,7 +225,7 @@ get_sys_stats() { if [[ -n "${ph_ver_raw[0]}" ]]; then ph_core_ver="${ph_ver_raw[0]}" if [[ ${#ph_ver_raw[@]} -eq 2 ]]; then - # AdminLTE not installed + # web not installed ph_lte_ver="(not installed)" ph_ftl_ver="${ph_ver_raw[1]}" else diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index b6153293..9dae66df 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -11,7 +11,7 @@ # Please see LICENSE file for your rights under this license. # Variables -readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/AdminLTE.git" +readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/web.git" readonly ADMIN_INTERFACE_DIR="/var/www/html/admin" readonly PI_HOLE_GIT_URL="https://github.com/pi-hole/pi-hole.git" readonly PI_HOLE_FILES_DIR="/etc/.pihole" diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 7d7103d2..8bb1888b 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -91,10 +91,10 @@ addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" WEB_HASH="$(get_local_hash /var/www/html/admin)" addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" -GITHUB_WEB_VERSION="$(get_remote_version AdminLTE)" +GITHUB_WEB_VERSION="$(get_remote_version web)" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" -GITHUB_WEB_HASH="$(get_remote_hash AdminLTE "${WEB_BRANCH}")" +GITHUB_WEB_HASH="$(get_remote_hash web "${WEB_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_HASH" "${GITHUB_WEB_HASH}" # get FTL versions diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index af86b045..e3b4a6ae 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -24,7 +24,7 @@ fi getLocalVersion() { case ${1} in "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "AdminLTE" ) echo "${WEB_VERSION:=N/A}";; + "web" ) echo "${WEB_VERSION:=N/A}";; "FTL" ) echo "${FTL_VERSION:=N/A}";; esac } @@ -32,7 +32,7 @@ getLocalVersion() { getLocalHash() { case ${1} in "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "AdminLTE" ) echo "${WEB_HASH:=N/A}";; + "web" ) echo "${WEB_HASH:=N/A}";; "FTL" ) echo "${FTL_HASH:=N/A}";; esac } @@ -40,7 +40,7 @@ getLocalHash() { getRemoteHash(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "AdminLTE" ) echo "${GITHUB_WEB_HASH:=N/A}";; + "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; esac } @@ -48,7 +48,7 @@ getRemoteHash(){ getRemoteVersion(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "AdminLTE" ) echo "${GITHUB_WEB_VERSION:=N/A}";; + "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; esac } @@ -56,7 +56,7 @@ getRemoteVersion(){ getLocalBranch(){ case ${1} in "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "AdminLTE" ) echo "${WEB_BRANCH:=N/A}";; + "web" ) echo "${WEB_BRANCH:=N/A}";; "FTL" ) echo "${FTL_BRANCH:=N/A}";; esac } @@ -107,7 +107,7 @@ errorOutput() { defaultOutput() { versionOutput "Pi-hole" "$@" - versionOutput "AdminLTE" "$@" + versionOutput "web" "$@" versionOutput "FTL" "$@" } @@ -118,7 +118,7 @@ Show Pi-hole, Admin Console & FTL versions Repositories: -p, --pihole Only retrieve info regarding Pi-hole repository - -a, --admin Only retrieve info regarding AdminLTE repository + -a, --admin Only retrieve info regarding web repository -f, --ftl Only retrieve info regarding FTL repository Options: @@ -131,7 +131,7 @@ Options: case "${1}" in "-p" | "--pihole" ) shift; versionOutput "Pi-hole" "$@";; - "-a" | "--admin" ) shift; versionOutput "AdminLTE" "$@";; + "-a" | "--admin" ) shift; versionOutput "web" "$@";; "-f" | "--ftl" ) shift; versionOutput "FTL" "$@";; "-h" | "--help" ) helpFunc;; * ) defaultOutput "$@";; diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4e4bdfc7..ef2c8d52 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -64,11 +64,11 @@ webroot="/var/www/html" # We clone (or update) two git repositories during the install. This helps to make sure that we always have the latest versions of the relevant files. -# AdminLTE is used to set up the Web admin interface. +# web is used to set up the Web admin interface. # Pi-hole contains various setup scripts and files which are critical to the installation. # Search for "PI_HOLE_LOCAL_REPO" in this file to see all such scripts. # Two notable scripts are gravity.sh (used to generate the HOSTS file) and advanced/Scripts/webpage.sh (used to install the Web admin interface) -webInterfaceGitUrl="https://github.com/pi-hole/AdminLTE.git" +webInterfaceGitUrl="https://github.com/pi-hole/web.git" webInterfaceDir="${webroot}/admin" piholeGitUrl="https://github.com/pi-hole/pi-hole.git" PI_HOLE_LOCAL_REPO="/etc/.pihole" diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 1cf8ab35..fec1fa5e 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -212,7 +212,7 @@ Available commands and options: .br -p, --pihole Only retrieve info regarding Pi-hole repository .br - -a, --admin Only retrieve info regarding AdminLTE + -a, --admin Only retrieve info regarding web repository .br -f, --ftl Only retrieve info regarding FTL repository @@ -339,7 +339,7 @@ Displaying version information \fBpihole -v -a -c\fR .br - Display the current version of AdminLTE + Display the current version of web .br Temporarily disabling Pi-hole From 71357ecae7900b880ad336a6583491595491a350 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 8 Oct 2023 14:12:02 +0100 Subject: [PATCH 193/462] adminlte->web Signed-off-by: Adam Warner --- README.md | 2 +- advanced/Scripts/chronometer.sh | 2 +- advanced/Scripts/update.sh | 2 +- advanced/Scripts/updatecheck.sh | 4 ++-- advanced/Scripts/version.sh | 18 +++++++++--------- advanced/Scripts/webpage.sh | 2 +- automated install/basic-install.sh | 6 +++--- manpages/pihole.8 | 4 ++-- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index adfd3450..eb50030b 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ You can read our [Core Feature Breakdown](https://docs.pi-hole.net/core/pihole-c ### The Web Interface Dashboard -This [optional dashboard](https://github.com/pi-hole/AdminLTE) allows you to view stats, change settings, and configure your Pi-hole. It's the power of the Command Line Interface, with none of the learning curve! +This [optional dashboard](https://github.com/pi-hole/web) allows you to view stats, change settings, and configure your Pi-hole. It's the power of the Command Line Interface, with none of the learning curve! Some notable features include: diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh index d69a56d3..bb94a857 100755 --- a/advanced/Scripts/chronometer.sh +++ b/advanced/Scripts/chronometer.sh @@ -233,7 +233,7 @@ get_sys_stats() { if [[ -n "${ph_ver_raw[0]}" ]]; then ph_core_ver="${ph_ver_raw[0]}" if [[ ${#ph_ver_raw[@]} -eq 2 ]]; then - # AdminLTE not installed + # web not installed ph_lte_ver="(not installed)" ph_ftl_ver="${ph_ver_raw[1]}" else diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index c41c9232..1b9997a6 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -11,7 +11,7 @@ # Please see LICENSE file for your rights under this license. # Variables -readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/AdminLTE.git" +readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/web.git" readonly ADMIN_INTERFACE_DIR="/var/www/html/admin" readonly PI_HOLE_GIT_URL="https://github.com/pi-hole/pi-hole.git" readonly PI_HOLE_FILES_DIR="/etc/.pihole" diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 0cc65218..8ce69a75 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -97,10 +97,10 @@ if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then WEB_HASH="$(get_local_hash /var/www/html/admin)" addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" - GITHUB_WEB_VERSION="$(get_remote_version AdminLTE)" + GITHUB_WEB_VERSION="$(get_remote_version web)" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" - GITHUB_WEB_HASH="$(get_remote_hash AdminLTE "${WEB_BRANCH}")" + GITHUB_WEB_HASH="$(get_remote_hash web "${WEB_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_HASH" "${GITHUB_WEB_HASH}" fi diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index 946c69fe..83fd0f39 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -28,7 +28,7 @@ fi getLocalVersion() { case ${1} in "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_VERSION:=N/A}";; + "web" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_VERSION:=N/A}";; "FTL" ) echo "${FTL_VERSION:=N/A}";; esac } @@ -36,7 +36,7 @@ getLocalVersion() { getLocalHash() { case ${1} in "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_HASH:=N/A}";; + "web" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_HASH:=N/A}";; "FTL" ) echo "${FTL_HASH:=N/A}";; esac } @@ -44,7 +44,7 @@ getLocalHash() { getRemoteHash(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_HASH:=N/A}";; + "web" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_HASH:=N/A}";; "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; esac } @@ -52,7 +52,7 @@ getRemoteHash(){ getRemoteVersion(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_VERSION:=N/A}";; + "web" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${GITHUB_WEB_VERSION:=N/A}";; "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; esac } @@ -60,13 +60,13 @@ getRemoteVersion(){ getLocalBranch(){ case ${1} in "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "AdminLTE" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_BRANCH:=N/A}";; + "web" ) [ "${INSTALL_WEB_INTERFACE}" = true ] && echo "${WEB_BRANCH:=N/A}";; "FTL" ) echo "${FTL_BRANCH:=N/A}";; esac } versionOutput() { - if [ "$1" = "AdminLTE" ] && [ "${INSTALL_WEB_INTERFACE}" != true ]; then + if [ "$1" = "web" ] && [ "${INSTALL_WEB_INTERFACE}" != true ]; then echo " WebAdmin not installed" return 1 fi @@ -117,7 +117,7 @@ defaultOutput() { versionOutput "Pi-hole" "$@" if [ "${INSTALL_WEB_INTERFACE}" = true ]; then - versionOutput "AdminLTE" "$@" + versionOutput "web" "$@" fi versionOutput "FTL" "$@" @@ -130,7 +130,7 @@ Show Pi-hole, Admin Console & FTL versions Repositories: -p, --pihole Only retrieve info regarding Pi-hole repository - -a, --admin Only retrieve info regarding AdminLTE repository + -a, --admin Only retrieve info regarding web repository -f, --ftl Only retrieve info regarding FTL repository Options: @@ -143,7 +143,7 @@ Options: case "${1}" in "-p" | "--pihole" ) shift; versionOutput "Pi-hole" "$@";; - "-a" | "--admin" ) shift; versionOutput "AdminLTE" "$@";; + "-a" | "--admin" ) shift; versionOutput "web" "$@";; "-f" | "--ftl" ) shift; versionOutput "FTL" "$@";; "-h" | "--help" ) helpFunc;; * ) defaultOutput "$@";; diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index c92d0458..e05a6aff 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -348,7 +348,7 @@ SetDNSServers() { IFS=',' read -r -a array <<< "${args[2]}" for index in "${!array[@]}" do - # Replace possible "\#" by "#". This fixes AdminLTE#1427 + # Replace possible "\#" by "#". This fixes web#1427 local ip ip="${array[index]//\\#/#}" diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e87edd71..4c69788f 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -68,11 +68,11 @@ webroot="/var/www/html" # We clone (or update) two git repositories during the install. This helps to make sure that we always have the latest versions of the relevant files. -# AdminLTE is used to set up the Web admin interface. +# web is used to set up the Web admin interface. # Pi-hole contains various setup scripts and files which are critical to the installation. # Search for "PI_HOLE_LOCAL_REPO" in this file to see all such scripts. # Two notable scripts are gravity.sh (used to generate the HOSTS file) and advanced/Scripts/webpage.sh (used to install the Web admin interface) -webInterfaceGitUrl="https://github.com/pi-hole/AdminLTE.git" +webInterfaceGitUrl="https://github.com/pi-hole/web.git" webInterfaceDir="${webroot}/admin" piholeGitUrl="https://github.com/pi-hole/pi-hole.git" PI_HOLE_LOCAL_REPO="/etc/.pihole" @@ -334,7 +334,7 @@ package_manager_detect() { # Packages required for the Web admin interface (stored as an array) # It's useful to separate this from Pi-hole, since the two repos are also setup separately PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") - # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole AdminLTE: https://www.php.net/manual/json.installation.php + # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole web: https://www.php.net/manual/json.installation.php if [[ -z "${phpInsMajor}" || "${phpInsMajor}" -lt 8 ]]; then PIHOLE_WEB_DEPS+=("${phpVer}-json") fi diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 1cf8ab35..fec1fa5e 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -212,7 +212,7 @@ Available commands and options: .br -p, --pihole Only retrieve info regarding Pi-hole repository .br - -a, --admin Only retrieve info regarding AdminLTE + -a, --admin Only retrieve info regarding web repository .br -f, --ftl Only retrieve info regarding FTL repository @@ -339,7 +339,7 @@ Displaying version information \fBpihole -v -a -c\fR .br - Display the current version of AdminLTE + Display the current version of web .br Temporarily disabling Pi-hole From 6001fe34ec3d3ad652a7013f467d1948efb3e096 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 8 Oct 2023 23:03:33 +0200 Subject: [PATCH 194/462] Set owner of gravity output files to pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gravity.sh b/gravity.sh index 9133d33d..4f87c1d7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -488,6 +488,10 @@ compareLists() { # We assume here it was changed upstream database_adlist_status "${adlistID}" "1" fi + + # set owner of the file to pihole + chown pihole:pihole "${target}.sha1" + } # Download specified URL and perform checks on HTTP status and file content @@ -621,6 +625,9 @@ gravity_DownloadBlocklistFromUrl() { database_adlist_status "${adlistID}" "4" fi fi + + # set owner of the file to pihole + chown pihole:pihole "${saveLocation}" } # Parse source files into domains format From 679aab10d0400d55470a5d53215158490736068a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 9 Oct 2023 21:52:43 +0200 Subject: [PATCH 195/462] Run gravity as user pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- gravity.sh | 7 ------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef2c8d52..ae605fcf 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1385,8 +1385,8 @@ installCron() { # Gravity is a very important script as it aggregates all of the domains into a single HOSTS formatted list, # which is what Pi-hole needs to begin blocking ads runGravity() { - # Run gravity in the current shell - { /opt/pihole/gravity.sh --force; } + # Run gravity in the current shell as user pihole + { exec sudo -u pihole bash /opt/pihole/gravity.sh --force; } } # Check if the pihole user exists and create if it does not diff --git a/gravity.sh b/gravity.sh index 4f87c1d7..9133d33d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -488,10 +488,6 @@ compareLists() { # We assume here it was changed upstream database_adlist_status "${adlistID}" "1" fi - - # set owner of the file to pihole - chown pihole:pihole "${target}.sha1" - } # Download specified URL and perform checks on HTTP status and file content @@ -625,9 +621,6 @@ gravity_DownloadBlocklistFromUrl() { database_adlist_status "${adlistID}" "4" fi fi - - # set owner of the file to pihole - chown pihole:pihole "${saveLocation}" } # Parse source files into domains format From 22863845a0018708aaf8b215cd00666e87fceebf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 9 Oct 2023 22:05:01 +0200 Subject: [PATCH 196/462] Set owner/group of /etc/pihole to pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ae605fcf..4a938dc7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2141,8 +2141,8 @@ main() { if [[ "${useUpdateVars}" == false ]]; then # Display welcome dialogs welcomeDialogs - # Create directory for Pi-hole storage - install -d -m 755 /etc/pihole/ + # Create directory for Pi-hole storage (/etc/pihole/) + install -o pihole -g pihole -d -m 660 "${PI_HOLE_CONFIG_DIR}" # Determine available interfaces get_available_interfaces # Find interfaces and let the user choose one From 2061f3a70e7f0e64709e330552ba8e85671080ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 9 Oct 2023 22:35:02 +0200 Subject: [PATCH 197/462] Set owner/permissions at the right place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4a938dc7..efe1d4f3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1085,12 +1085,15 @@ installScripts() { installConfigs() { printf "\\n %b Installing configs from %s...\\n" "${INFO}" "${PI_HOLE_LOCAL_REPO}" + # Ensure that permissions are correctly set + chown -R pihole:pihole /etc/pihole # Install list of DNS servers # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6 # Some values may be empty (for example: DNS servers without IPv6 support) echo "${DNS_SERVERS}" > "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" + chown pihole:pihole "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" # Install empty custom.list file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then @@ -1386,7 +1389,7 @@ installCron() { # which is what Pi-hole needs to begin blocking ads runGravity() { # Run gravity in the current shell as user pihole - { exec sudo -u pihole bash /opt/pihole/gravity.sh --force; } + { sudo -u pihole bash /opt/pihole/gravity.sh --force; } } # Check if the pihole user exists and create if it does not @@ -1480,7 +1483,7 @@ installLogrotate() { return 2 fi # Copy the file over from the local repo - install -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} + install -o pihole -g pihole -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. @@ -2049,6 +2052,7 @@ copy_to_install_log() { # Since we use color codes such as '\e[1;33m', they should be removed sed 's/\[[0-9;]\{1,5\}m//g' < /proc/$$/fd/3 > "${installLogLoc}" chmod 644 "${installLogLoc}" + chown pihole:pihole "${installLogLoc}" } main() { @@ -2142,7 +2146,7 @@ main() { # Display welcome dialogs welcomeDialogs # Create directory for Pi-hole storage (/etc/pihole/) - install -o pihole -g pihole -d -m 660 "${PI_HOLE_CONFIG_DIR}" + install -d -m 755 "${PI_HOLE_CONFIG_DIR}" # Determine available interfaces get_available_interfaces # Find interfaces and let the user choose one From 46ff25734485c9f960da0c4312a6c5ebc5e0f8a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 12 Oct 2023 13:44:51 +0200 Subject: [PATCH 198/462] Remove Chronometer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/chronometer.sh | 569 ----------------------------- advanced/bash-completion/pihole | 8 +- automated install/basic-install.sh | 2 +- manpages/pihole.8 | 14 - pihole | 6 +- 5 files changed, 5 insertions(+), 594 deletions(-) delete mode 100755 advanced/Scripts/chronometer.sh diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh deleted file mode 100755 index fc728e17..00000000 --- a/advanced/Scripts/chronometer.sh +++ /dev/null @@ -1,569 +0,0 @@ -#!/usr/bin/env bash -# shellcheck disable=SC1090,SC1091 -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Calculates stats and displays to an LCD -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. -LC_ALL=C -LC_NUMERIC=C - -# Retrieve stats from FTL engine -pihole-FTL() { - local ftl_port LINE - # shellcheck disable=SC1091 - . /opt/pihole/utils.sh - ftl_port=$(getFTLConfigValue dns.port) - if [[ -n "$ftl_port" ]]; then - # Open connection to FTL - exec 3<>"/dev/tcp/127.0.0.1/$ftl_port" - - # Test if connection is open - if { "true" >&3; } 2> /dev/null; then - # Send command to FTL and ask to quit when finished - echo -e ">$1 >quit" >&3 - - # Read input until we received an empty string and the connection is - # closed - read -r -t 1 LINE <&3 - until [[ -z "${LINE}" ]] && [[ ! -t 3 ]]; do - echo "$LINE" >&1 - read -r -t 1 LINE <&3 - done - - # Close connection - exec 3>&- - exec 3<&- - fi - else - echo "0" - fi -} - -# Print spaces to align right-side additional text -printFunc() { - local text_last - - title="$1" - title_len="${#title}" - - text_main="$2" - text_main_nocol="$text_main" - if [[ "${text_main:0:1}" == "" ]]; then - text_main_nocol=$(sed 's/\[[0-9;]\{1,5\}m//g' <<< "$text_main") - fi - text_main_len="${#text_main_nocol}" - - text_addn="$3" - if [[ "$text_addn" == "last" ]]; then - text_addn="" - text_last="true" - fi - - # If there is additional text, define max length of text_main - if [[ -n "$text_addn" ]]; then - case "$scr_cols" in - [0-9]|1[0-9]|2[0-9]|3[0-9]|4[0-4]) text_main_max_len="9";; - 4[5-9]) text_main_max_len="14";; - *) text_main_max_len="19";; - esac - fi - - [[ -z "$text_addn" ]] && text_main_max_len="$(( scr_cols - title_len ))" - - # Remove excess characters from main text - if [[ "$text_main_len" -gt "$text_main_max_len" ]]; then - # Trim text without colors - text_main_trim="${text_main_nocol:0:$text_main_max_len}" - # Replace with trimmed text - text_main="${text_main/$text_main_nocol/$text_main_trim}" - fi - - # Determine amount of spaces for each line - if [[ -n "$text_last" ]]; then - # Move cursor to end of screen - spc_num=$(( scr_cols - ( title_len + text_main_len ) )) - else - spc_num=$(( text_main_max_len - text_main_len )) - fi - - [[ "$spc_num" -le 0 ]] && spc_num="0" - spc=$(printf "%${spc_num}s") - #spc="${spc// /.}" # Debug: Visualize spaces - - printf "%s%s$spc" "$title" "$text_main" - - if [[ -n "$text_addn" ]]; then - printf "%s(%s)%s\\n" "$COL_NC$COL_DARK_GRAY" "$text_addn" "$COL_NC" - else - # Do not print trailing newline on final line - [[ -z "$text_last" ]] && printf "%s\\n" "$COL_NC" - fi -} - -# Perform on first Chrono run (not for JSON formatted string) -get_init_stats() { - calcFunc(){ awk "BEGIN {print $*}" 2> /dev/null; } - - # Convert bytes to human-readable format - hrBytes() { - awk '{ - num=$1; - if(num==0) { - print "0 B" - } else { - xxx=(num<0?-num:num) - sss=(num<0?-1:1) - split("B KB MB GB TB PB",type) - for(i=5;yyy < 1;i--) { - yyy=xxx / (2^(10*i)) - } - printf "%.0f " type[i+2], yyy*sss - } - }' <<< "$1"; - } - - # Convert seconds to human-readable format - hrSecs() { - day=$(( $1/60/60/24 )); hrs=$(( $1/3600%24 )) - mins=$(( ($1%3600)/60 )); secs=$(( $1%60 )) - [[ "$day" -ge "2" ]] && plu="s" - [[ "$day" -ge "1" ]] && days="$day day${plu}, " || days="" - printf "%s%02d:%02d:%02d\\n" "$days" "$hrs" "$mins" "$secs" - } - - # Set Color Codes - coltable="/opt/pihole/COL_TABLE" - if [[ -f "${coltable}" ]]; then - source ${coltable} - else - COL_NC="" - COL_DARK_GRAY="" - COL_LIGHT_GREEN="" - COL_LIGHT_BLUE="" - COL_LIGHT_RED="" - COL_YELLOW="" - COL_LIGHT_RED="" - COL_URG_RED="" - fi - - # Get RPi throttle state (RPi 3B only) & model number, or OS distro info - if command -v vcgencmd &> /dev/null; then - local sys_throttle_raw - local sys_rev_raw - - sys_throttle_raw=$(vgt=$(sudo vcgencmd get_throttled); echo "${vgt##*x}") - - # Active Throttle Notice: https://bit.ly/2gnunOo - if [[ "$sys_throttle_raw" != "0" ]]; then - case "$sys_throttle_raw" in - *0001) thr_type="${COL_YELLOW}Under Voltage";; - *0002) thr_type="${COL_LIGHT_BLUE}Arm Freq Cap";; - *0003) thr_type="${COL_YELLOW}UV${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_BLUE}AFC";; - *0004) thr_type="${COL_LIGHT_RED}Throttled";; - *0005) thr_type="${COL_YELLOW}UV${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_RED}TT";; - *0006) thr_type="${COL_LIGHT_BLUE}AFC${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_RED}TT";; - *0007) thr_type="${COL_YELLOW}UV${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_BLUE}AFC${COL_DARK_GRAY},${COL_NC} ${COL_LIGHT_RED}TT";; - esac - [[ -n "$thr_type" ]] && sys_throttle="$thr_type${COL_DARK_GRAY}" - fi - - sys_rev_raw=$(awk '/Revision/ {print $3}' < /proc/cpuinfo) - case "$sys_rev_raw" in - 000[2-6]) sys_model=" 1, Model B";; # 256MB - 000[7-9]) sys_model=" 1, Model A";; # 256MB - 000d|000e|000f) sys_model=" 1, Model B";; # 512MB - 0010|0013) sys_model=" 1, Model B+";; # 512MB - 0012|0015) sys_model=" 1, Model A+";; # 256MB - a0104[0-1]|a21041|a22042) sys_model=" 2, Model B";; # 1GB - 900021) sys_model=" 1, Model A+";; # 512MB - 900032) sys_model=" 1, Model B+";; # 512MB - 90009[2-3]|920093) sys_model=" Zero";; # 512MB - 9000c1) sys_model=" Zero W";; # 512MB - a02082|a[2-3]2082) sys_model=" 3, Model B";; # 1GB - a020d3) sys_model=" 3, Model B+";; # 1GB - *) sys_model="";; - esac - sys_type="Raspberry Pi$sys_model" - else - source "/etc/os-release" - CODENAME=$(sed 's/[()]//g' <<< "${VERSION/* /}") - sys_type="${NAME/ */} ${CODENAME^} $VERSION_ID" - fi - - # Get core count - sys_cores=$(grep -c "^processor" /proc/cpuinfo) - - # Test existence of clock speed file for ARM CPU - if [[ -f "/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq" ]]; then - scaling_freq_file="/sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq" - fi - - # Test existence of temperature file - if [[ -f "/sys/class/thermal/thermal_zone0/temp" ]]; then - temp_file="/sys/class/thermal/thermal_zone0/temp" - elif [[ -f "/sys/class/hwmon/hwmon0/temp1_input" ]]; then - temp_file="/sys/class/hwmon/hwmon0/temp1_input" - else - temp_file="" - fi -} - -get_sys_stats() { - local ph_ver_raw - local cpu_raw - local ram_raw - local disk_raw - - # Update every 12 refreshes (Def: every 60s) - count=$((count+1)) - if [[ "$count" == "1" ]] || (( "$count" % 12 == 0 )); then - mapfile -t ph_ver_raw < <(pihole -v -c 2> /dev/null | sed -n 's/^.* v/v/p') - if [[ -n "${ph_ver_raw[0]}" ]]; then - ph_core_ver="${ph_ver_raw[0]}" - if [[ ${#ph_ver_raw[@]} -eq 2 ]]; then - # web not installed - ph_lte_ver="(not installed)" - ph_ftl_ver="${ph_ver_raw[1]}" - else - ph_lte_ver="${ph_ver_raw[1]}" - ph_ftl_ver="${ph_ver_raw[2]}" - fi - else - ph_core_ver="-1" - fi - - sys_name=$(hostname) - - [[ -n "$TEMPERATUREUNIT" ]] && temp_unit="${TEMPERATUREUNIT^^}" || temp_unit="C" - - # Get storage stats for partition mounted on / - read -r -a disk_raw <<< "$(df -B1 / 2> /dev/null | awk 'END{ print $3,$2,$5 }')" - disk_used="${disk_raw[0]}" - disk_total="${disk_raw[1]}" - disk_perc="${disk_raw[2]}" - - net_gateway=$(ip route | grep default | cut -d ' ' -f 3 | head -n 1) - - # Get DHCP stats, if feature is enabled - if [[ "$DHCP_ACTIVE" == "true" ]]; then - ph_dhcp_max=$(( ${DHCP_END##*.} - ${DHCP_START##*.} + 1 )) - fi - - # Get DNS server count - dns_count="0" - [[ -n "${PIHOLE_DNS_1}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_2}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_3}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_4}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_5}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_6}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_7}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_8}" ]] && dns_count=$((dns_count+1)) - [[ -n "${PIHOLE_DNS_9}" ]] && dns_count="$dns_count+" - fi - - # Get screen size - read -r -a scr_size <<< "$(stty size 2>/dev/null || echo 24 80)" - scr_lines="${scr_size[0]}" - scr_cols="${scr_size[1]}" - - # Determine Chronometer size behavior - if [[ "$scr_cols" -ge 58 ]]; then - chrono_width="large" - elif [[ "$scr_cols" -gt 40 ]]; then - chrono_width="medium" - else - chrono_width="small" - fi - - # Determine max length of divider string - scr_line_len=$(( scr_cols - 2 )) - [[ "$scr_line_len" -ge 58 ]] && scr_line_len="58" - scr_line_str=$(printf "%${scr_line_len}s") - scr_line_str="${scr_line_str// /—}" - - sys_uptime=$(hrSecs "$(cut -d. -f1 /proc/uptime)") - sys_loadavg=$(cut -d " " -f1,2,3 /proc/loadavg) - - # Get CPU usage, only counting processes over 1% as active - # shellcheck disable=SC2009 - cpu_raw=$(ps -eo pcpu,rss --no-headers | grep -E -v " 0") - cpu_tasks=$(wc -l <<< "$cpu_raw") - cpu_taskact=$(sed -r "/(^ 0.)/d" <<< "$cpu_raw" | wc -l) - cpu_perc=$(awk '{sum+=$1} END {printf "%.0f\n", sum/'"$sys_cores"'}' <<< "$cpu_raw") - - # Get CPU clock speed - if [[ -n "$scaling_freq_file" ]]; then - cpu_mhz=$(( $(< /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq) / 1000 )) - else - cpu_mhz=$(lscpu | awk -F ":" '/MHz/ {print $2;exit}') - cpu_mhz=$(printf "%.0f" "${cpu_mhz//[[:space:]]/}") - fi - - # Determine whether to display CPU clock speed as MHz or GHz - if [[ -n "$cpu_mhz" ]]; then - [[ "$cpu_mhz" -le "999" ]] && cpu_freq="$cpu_mhz MHz" || cpu_freq="$(printf "%.1f" $(calcFunc "$cpu_mhz"/1000)) GHz" - [[ "${cpu_freq}" == *".0"* ]] && cpu_freq="${cpu_freq/.0/}" - fi - - # Determine color for temperature - if [[ -n "$temp_file" ]]; then - if [[ "$temp_unit" == "C" ]]; then - cpu_temp=$(printf "%.0fc\\n" "$(calcFunc "$(< $temp_file) / 1000")") - - case "${cpu_temp::-1}" in - -*|[0-9]|[1-3][0-9]) cpu_col="$COL_LIGHT_BLUE";; - 4[0-9]) cpu_col="";; - 5[0-9]) cpu_col="$COL_YELLOW";; - 6[0-9]) cpu_col="$COL_LIGHT_RED";; - *) cpu_col="$COL_URG_RED";; - esac - - # $COL_NC$COL_DARK_GRAY is needed for $COL_URG_RED - cpu_temp_str=" @ $cpu_col$cpu_temp$COL_NC$COL_DARK_GRAY" - - elif [[ "$temp_unit" == "F" ]]; then - cpu_temp=$(printf "%.0ff\\n" "$(calcFunc "($(< $temp_file) / 1000) * 9 / 5 + 32")") - - case "${cpu_temp::-1}" in - -*|[0-9]|[0-9][0-9]) cpu_col="$COL_LIGHT_BLUE";; - 1[0-1][0-9]) cpu_col="";; - 1[2-3][0-9]) cpu_col="$COL_YELLOW";; - 1[4-5][0-9]) cpu_col="$COL_LIGHT_RED";; - *) cpu_col="$COL_URG_RED";; - esac - - cpu_temp_str=" @ $cpu_col$cpu_temp$COL_NC$COL_DARK_GRAY" - - else - cpu_temp_str=$(printf " @ %.0fk\\n" "$(calcFunc "($(< $temp_file) / 1000) + 273.15")") - fi - else - cpu_temp_str="" - fi - - read -r -a ram_raw <<< "$(awk '/MemTotal:/{total=$2} /MemFree:/{free=$2} /Buffers:/{buffers=$2} /^Cached:/{cached=$2} END {printf "%.0f %.0f %.0f", (total-free-buffers-cached)*100/total, (total-free-buffers-cached)*1024, total*1024}' /proc/meminfo)" - ram_perc="${ram_raw[0]}" - ram_used="${ram_raw[1]}" - ram_total="${ram_raw[2]}" - - if [[ "$(pihole status web 2> /dev/null)" -ge "1" ]]; then - ph_status="${COL_LIGHT_GREEN}Active" - else - ph_status="${COL_LIGHT_RED}Offline" - fi - - if [[ "$DHCP_ACTIVE" == "true" ]]; then - local ph_dhcp_range - - ph_dhcp_range=$(seq -s "|" -f "${DHCP_START%.*}.%g" "${DHCP_START##*.}" "${DHCP_END##*.}") - - # Count dynamic leases from available range, and not static leases - ph_dhcp_num=$(grep -cE "$ph_dhcp_range" "/etc/pihole/dhcp.leases") - ph_dhcp_percent=$(( ph_dhcp_num * 100 / ph_dhcp_max )) - fi -} - -get_ftl_stats() { - local stats_raw - - mapfile -t stats_raw < <(pihole-FTL "stats") - domains_being_blocked_raw="${stats_raw[0]#* }" - dns_queries_today_raw="${stats_raw[1]#* }" - ads_blocked_today_raw="${stats_raw[2]#* }" - ads_percentage_today_raw="${stats_raw[3]#* }" - queries_forwarded_raw="${stats_raw[5]#* }" - queries_cached_raw="${stats_raw[6]#* }" - - # Only retrieve these stats when not called from jsonFunc - if [[ -z "$1" ]]; then - local top_ad_raw - local top_domain_raw - local top_client_raw - - domains_being_blocked=$(printf "%.0f\\n" "${domains_being_blocked_raw}" 2> /dev/null) - dns_queries_today=$(printf "%.0f\\n" "${dns_queries_today_raw}") - ads_blocked_today=$(printf "%.0f\\n" "${ads_blocked_today_raw}") - ads_percentage_today=$(printf "%'.0f\\n" "${ads_percentage_today_raw}") - queries_cached_percentage=$(printf "%.0f\\n" "$(calcFunc "$queries_cached_raw * 100 / ( $queries_forwarded_raw + $queries_cached_raw )")") - recent_blocked=$(pihole-FTL recentBlocked) - read -r -a top_ad_raw <<< "$(pihole-FTL "top-ads (1)")" - read -r -a top_domain_raw <<< "$(pihole-FTL "top-domains (1)")" - read -r -a top_client_raw <<< "$(pihole-FTL "top-clients (1)")" - - top_ad="${top_ad_raw[2]}" - top_domain="${top_domain_raw[2]}" - if [[ "${top_client_raw[3]}" ]]; then - top_client="${top_client_raw[3]}" - else - top_client="${top_client_raw[2]}" - fi - fi -} - -get_strings() { - # Expand or contract strings depending on screen size - if [[ "$chrono_width" == "large" ]]; then - phc_str=" ${COL_DARK_GRAY}Core" - lte_str=" ${COL_DARK_GRAY}Web" - ftl_str=" ${COL_DARK_GRAY}FTL" - api_str="${COL_LIGHT_RED}API Offline" - - host_info="$sys_type" - sys_info="$sys_throttle" - sys_info2="Active: $cpu_taskact of $cpu_tasks tasks" - used_str="Used: " - leased_str="Leased: " - domains_being_blocked=$(printf "%'.0f" "$domains_being_blocked") - ads_blocked_today=$(printf "%'.0f" "$ads_blocked_today") - dns_queries_today=$(printf "%'.0f" "$dns_queries_today") - ph_info="Blocking: $domains_being_blocked sites" - total_str="Total: " - else - phc_str=" ${COL_DARK_GRAY}Core" - lte_str=" ${COL_DARK_GRAY}Web" - ftl_str=" ${COL_DARK_GRAY}FTL" - api_str="${COL_LIGHT_RED}API Down" - ph_info="$domains_being_blocked blocked" - fi - - [[ "$sys_cores" -ne 1 ]] && sys_cores_txt="${sys_cores}x " - cpu_info="$sys_cores_txt$cpu_freq$cpu_temp_str" - ram_info="$used_str$(hrBytes "$ram_used") of $(hrBytes "$ram_total")" - disk_info="$used_str$(hrBytes "$disk_used") of $(hrBytes "$disk_total")" - - lan_info="Gateway: $net_gateway" - dhcp_info="$leased_str$ph_dhcp_num of $ph_dhcp_max" - - ads_info="$total_str$ads_blocked_today of $dns_queries_today" - dns_info="$dns_count DNS servers" - - [[ "$recent_blocked" == "0" ]] && recent_blocked="${COL_LIGHT_RED}FTL offline${COL_NC}" -} - -chronoFunc() { - local extra_arg="$1" - local extra_value="$2" - - get_init_stats - - for (( ; ; )); do - get_sys_stats - get_ftl_stats - get_strings - - # Strip excess development version numbers - if [[ "$ph_core_ver" != "-1" ]]; then - phc_ver_str="$phc_str: ${ph_core_ver%-*}${COL_NC}" - lte_ver_str="$lte_str: ${ph_lte_ver%-*}${COL_NC}" - ftl_ver_str="$ftl_str: ${ph_ftl_ver%-*}${COL_NC}" - else - phc_ver_str="$phc_str: $api_str${COL_NC}" - fi - - # Get refresh number - if [[ "${extra_arg}" = "refresh" ]]; then - num="${extra_value}" - num_str="Refresh set for every $num seconds" - else - num_str="" - fi - - clear - - # Remove exit message heading on third refresh - if [[ "$count" -le 2 ]] && [[ "${extra_arg}" != "exit" ]]; then - echo -e " ${COL_LIGHT_GREEN}Pi-hole Chronometer${COL_NC} - $num_str - ${COL_LIGHT_RED}Press Ctrl-C to exit${COL_NC} - ${COL_DARK_GRAY}$scr_line_str${COL_NC}" - else - echo -e "|¯¯¯(¯)_|¯|_ ___|¯|___$phc_ver_str\\n| ¯_/¯|_| ' \\/ _ \\ / -_)$lte_ver_str\\n|_| |_| |_||_\\___/_\\___|$ftl_ver_str\\n ${COL_DARK_GRAY}$scr_line_str${COL_NC}" - fi - - printFunc " Hostname: " "$sys_name" "$host_info" - printFunc " Uptime: " "$sys_uptime" "$sys_info" - printFunc " Task Load: " "$sys_loadavg" "$sys_info2" - printFunc " CPU usage: " "$cpu_perc%" "$cpu_info" - printFunc " RAM usage: " "$ram_perc%" "$ram_info" - printFunc " HDD usage: " "$disk_perc" "$disk_info" - - if [[ "$DHCP_ACTIVE" == "true" ]]; then - printFunc "DHCP usage: " "$ph_dhcp_percent%" "$dhcp_info" - fi - - printFunc " Pi-hole: " "$ph_status" "$ph_info" - printFunc " Blocked: " "$ads_percentage_today%" "$ads_info" - printFunc "Local Qrys: " "$queries_cached_percentage%" "$dns_info" - - printFunc "Last Block: " "$recent_blocked" - printFunc " Top Block: " "$top_ad" - - # Provide more stats on screens with more lines - if [[ "$scr_lines" -eq 17 ]]; then - if [[ "$DHCP_ACTIVE" == "true" ]]; then - printFunc "Top Domain: " "$top_domain" "last" - else - print_client="true" - fi - else - print_client="true" - fi - - if [[ -n "$print_client" ]]; then - printFunc "Top Domain: " "$top_domain" - printFunc "Top Client: " "$top_client" "last" - fi - - # Handle exit/refresh options - if [[ "${extra_arg}" == "exit" ]]; then - exit 0 - else - if [[ "${extra_arg}" == "refresh" ]]; then - sleep "$num" - else - sleep 5 - fi - fi - - done -} - -jsonFunc() { - get_ftl_stats "json" - echo "{\"domains_being_blocked\":${domains_being_blocked_raw},\"dns_queries_today\":${dns_queries_today_raw},\"ads_blocked_today\":${ads_blocked_today_raw},\"ads_percentage_today\":${ads_percentage_today_raw}}" -} - -helpFunc() { - if [[ "$1" == "?" ]]; then - echo "Unknown option. Please view 'pihole -c --help' for more information" - else - echo "Usage: pihole -c [options] -Example: 'pihole -c -j' -Calculates stats and displays to an LCD - -Options: - -j, --json Output stats as JSON formatted string - -r, --refresh Set update frequency (in seconds) - -e, --exit Output stats and exit without refreshing - -h, --help Display this help text" - fi - - exit 0 -} - -if [[ $# = 0 ]]; then - chronoFunc -fi - -case "$1" in - "-j" | "--json" ) jsonFunc;; - "-h" | "--help" ) helpFunc;; - "-r" | "--refresh" ) chronoFunc refresh "$2";; - "-e" | "--exit" ) chronoFunc exit;; - * ) helpFunc "?";; -esac diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 29a3270d..305a3f5b 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -1,5 +1,5 @@ _pihole() { - local cur prev opts opts_admin opts_checkout opts_chronometer opts_debug opts_interface opts_logging opts_privacy opts_query opts_update opts_version + local cur prev opts opts_admin opts_checkout opts_debug opts_interface opts_logging opts_privacy opts_query opts_update opts_version COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" @@ -7,7 +7,7 @@ _pihole() { case "${prev}" in "pihole") - opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" + opts="admin blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; "whitelist"|"blacklist"|"wildcard"|"regex") @@ -22,10 +22,6 @@ _pihole() { opts_checkout="core ftl web master dev" COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) ;; - "chronometer") - opts_chronometer="\--exit \--json \--refresh" - COMPREPLY=( $(compgen -W "${opts_chronometer}" -- ${cur}) ) - ;; "debug") opts_debug="-a" COMPREPLY=( $(compgen -W "${opts_debug}" -- ${cur}) ) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef2c8d52..cc187499 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -73,7 +73,7 @@ webInterfaceDir="${webroot}/admin" piholeGitUrl="https://github.com/pi-hole/pi-hole.git" PI_HOLE_LOCAL_REPO="/etc/.pihole" # List of pihole scripts, stored in an array -PI_HOLE_FILES=(chronometer list piholeDebug piholeLogFlush setupLCD update version gravity uninstall webpage) +PI_HOLE_FILES=(list piholeDebug piholeLogFlush setupLCD update version gravity uninstall webpage) # This directory is where the Pi-hole scripts will be installed PI_HOLE_INSTALL_DIR="/opt/pihole" PI_HOLE_CONFIG_DIR="/etc/pihole" diff --git a/manpages/pihole.8 b/manpages/pihole.8 index fec1fa5e..55bbe6cb 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -141,20 +141,6 @@ Available commands and options: (0 = lowest, 3 = highest) .br -\fB-c, chronometer\fR [options] -.br - Calculates stats and displays to an LCD -.br - - (Chronometer Options): -.br - -j, --json Output stats as JSON formatted string -.br - -r, --refresh Set update frequency (in seconds) -.br - -e, --exit Output stats and exit without refreshing -.br - \fB-g, updateGravity\fR .br Update the list of ad-serving domains diff --git a/pihole b/pihole index 5281c715..817bfd01 100755 --- a/pihole +++ b/pihole @@ -125,8 +125,7 @@ queryFunc() { } chronometerFunc() { - shift - "${PI_HOLE_SCRIPT_DIR}"/chronometer.sh "$@" + echo "Chronometer is gone, use PADD (https://github.com/pi-hole/PADD)" exit 0 } @@ -497,8 +496,7 @@ Debugging Options: Options: setpassword set the password for the web interface - -c, chronometer Calculates stats and displays to an LCD - Add '-h' for more info on chronometer usage + -c, chronometer Chronometer is gone, use PADD -g, updateGravity Update the list of ad-serving domains -h, --help, help Show this help dialog -l, logging Specify whether the Pi-hole log should be used From 188b2b858acb907c2a856bbe032db833db9df75e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 12 Oct 2023 13:48:01 +0200 Subject: [PATCH 199/462] PADD is fine, really! MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .codespellignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.codespellignore b/.codespellignore index 501a3d67..0755931d 100644 --- a/.codespellignore +++ b/.codespellignore @@ -1,3 +1,4 @@ doubleclick wan nwe +padd From 40c75289b5c88bea75bf3803729a1ecf49b4b2d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 13 Oct 2023 19:59:29 +0200 Subject: [PATCH 200/462] Allow pihole to access subdirs in /etc/pihole MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index 17900f0b..abeaabc4 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -9,20 +9,27 @@ utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" # Get file paths FTL_PID_FILE="$(getFTLPIDFile)" -# Touch files to ensure they exist (create if non-existing, preserve if existing) -# shellcheck disable=SC2174 -mkdir -pm 0755 /var/log/pihole -[ -f "${FTL_PID_FILE}" ] || install -D -m 644 -o pihole -g pihole /dev/null "${FTL_PID_FILE}" -[ -f /var/log/pihole/FTL.log ] || install -m 644 -o pihole -g pihole /dev/null /var/log/pihole/FTL.log -[ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log -[ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files +# shellcheck disable=SC2174 +mkdir -pm 0640 /var/log/pihole chown -R pihole:pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole chmod -R 0660 /etc/pihole + # allow all users to enter the directories chmod 0755 /etc/pihole /var/log/pihole +# allow pihole to access subdirs in /etc/pihole (sets execution bit on dirs) +# credits https://stackoverflow.com/a/11512211 +find /etc/pihole -type d -exec chmod 0755 {} \; + +# Touch files to ensure they exist (create if non-existing, preserve if existing) +[ -f "${FTL_PID_FILE}" ] || install -D -m 644 -o pihole -g pihole /dev/null "${FTL_PID_FILE}" +[ -f /var/log/pihole/FTL.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/FTL.log +[ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log +[ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases + + # Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole # Should be removed with Pi-hole v6.0 if [ ! -f /var/log/pihole.log ]; then From f7ba059b94215160731d4e30fbc239c375c104ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 14 Oct 2023 14:01:20 +0200 Subject: [PATCH 201/462] Query directly and authenticate only if required MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 21 ++++++++++++++++++--- advanced/Scripts/query.sh | 15 +++++++++++++-- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index afd88671..449f146f 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -82,10 +82,25 @@ DeleteSession() { } GetFTLData() { - local data + local data response status # get the data from querying the API as well as the http status code - data=$(curl -s -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) - echo "${data}" + response=$(curl -s -w "%{http_code}" -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) + + # status are the last 3 characters + status=$(printf %s "${response#"${response%???}"}") + # data is everything from response without the last 3 characters + data=$(printf %s "${response%???}") + + if [ "${status}" = 200 ]; then + # response OK + echo "${data}" + elif [ "${status}" = 000 ]; then + # connection lost + echo "000" + elif [ "${status}" = 401 ]; then + # unauthorized + echo "401" + fi } secretRead() { diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 8407d3f6..a26d249c 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -118,10 +118,21 @@ Main(){ # Test if the authentication endpoint is available TestAPIAvailability - # Authenticate with the FTL server - Authenthication + + # Users can configure FTL in a way, that for accessing a) all endpoints (webserver.api.localAPIauth) + # or b) for the /search endpoint (webserver.api.searchAPIauth) no authentication is required. + # Therefore, we try to query directly without authentication but do authenticat if 401 is returned data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + + if [ "${data}" = 401 ]; then + # Unauthenticated, so authenticate with the FTL server required + Authenthication + + # send query again + data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + fi + GenerateOutput "${data}" DeleteSession } From ec83d6b7931dc3493efb535ce8c66586bec53d7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 14 Oct 2023 14:06:47 +0200 Subject: [PATCH 202/462] No help for gone functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 1 - 1 file changed, 1 deletion(-) diff --git a/pihole b/pihole index 817bfd01..7be13a6f 100755 --- a/pihole +++ b/pihole @@ -496,7 +496,6 @@ Debugging Options: Options: setpassword set the password for the web interface - -c, chronometer Chronometer is gone, use PADD -g, updateGravity Update the list of ad-serving domains -h, --help, help Show this help dialog -l, logging Specify whether the Pi-hole log should be used From 2005d04625c48948c1010563dac3a060f114f311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 14 Oct 2023 21:50:57 +0200 Subject: [PATCH 203/462] Exact search is the new default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/query.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index a26d249c..27a33f39 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -17,7 +17,7 @@ # Globals PI_HOLE_INSTALL_DIR="/opt/pihole" max_results="20" -partial="true" +partial="false" domain="" # Source color table @@ -29,13 +29,13 @@ colfile="/opt/pihole/COL_TABLE" Help(){ echo "Usage: pihole -q [option] -Example: 'pihole -q --exact domain.com' +Example: 'pihole -q --partial domain.com' Query the adlists for a specified domain Options: - --exact Search the adlists for exact domain matches + --partial Search the adlists for partially matching domains --all Return all query matches within the adlists - -h, --help Show this help dialog" + -h, --help Show this help dialog" exit 0 } @@ -141,7 +141,7 @@ Main(){ while [ "$#" -gt 0 ]; do case "$1" in "-h" | "--help" ) Help;; - "--exact" ) partial="false";; + "--partial" ) partial="true";; "--all" ) max_results=10000;; # hard-coded FTL limit * ) domain=$1;; esac From 123ba1f154845ad9b549f3f81638c3ef340dbda6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 15 Oct 2023 22:15:11 +0200 Subject: [PATCH 204/462] Remove temp dir created when downloading FTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ac516415..4dd31976 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1777,7 +1777,8 @@ FTLinstall() { # Move into the temp ftl directory pushd "$(mktemp -d)" > /dev/null || { printf "Unable to make temporary directory for FTL binary download\\n"; return 1; } - + local tempdir + tempdir="$(pwd)" local ftlBranch local url @@ -1819,12 +1820,19 @@ FTLinstall() { # Installed the FTL service printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" + + # Remove temp dir + remove_dir "${tempdir}" + return 0 else # Otherwise, the hash download failed, so print and exit. popd > /dev/null || { printf "Unable to return to original directory after FTL binary download.\\n"; return 1; } printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" printf " %b Error: Download of %s/%s failed (checksum error)%b\\n" "${COL_LIGHT_RED}" "${url}" "${binary}" "${COL_NC}" + + # Remove temp dir + remove_dir "${tempdir}" return 1 fi else @@ -1833,10 +1841,19 @@ FTLinstall() { printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" # The URL could not be found printf " %b Error: URL %s/%s not found%b\\n" "${COL_LIGHT_RED}" "${url}" "${binary}" "${COL_NC}" + + # Remove temp dir + remove_dir "${tempdir}" return 1 fi } +remove_dir() { + # Delete dir + rm -r "${1}" > /dev/null 2>&1 || \ + echo -e " ${CROSS} Unable to remove ${1}" +} + get_binary_name() { # Get the OS architecture (we cannot use uname -m as this may return an incorrect architecture when buildx-compiling with QEMU for arm) local machine From 27522fbc18dc53cd0828778d5b7a8a2dfd72ff6d Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Mon, 16 Oct 2023 17:19:44 -0300 Subject: [PATCH 205/462] Add a final message to gravity The terminal version doesn't really need a final message, but this will be read by the web interface to show a success message Signed-off-by: RD WebDesign --- gravity.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gravity.sh b/gravity.sh index 9133d33d..e61ade42 100755 --- a/gravity.sh +++ b/gravity.sh @@ -901,4 +901,6 @@ gravity_ShowCount gravity_Cleanup echo "" +echo " ${TICK} Done." + # "${PIHOLE_COMMAND}" status From 2c7fa4a7b36aa4ae440ea5500438f72e56bb0f91 Mon Sep 17 00:00:00 2001 From: RD WebDesign Date: Wed, 18 Oct 2023 02:07:06 -0300 Subject: [PATCH 206/462] Avoid printing getFTLConfigValue return in statusFunc() Signed-off-by: RD WebDesign --- pihole | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pihole b/pihole index 7be13a6f..9cc1069f 100755 --- a/pihole +++ b/pihole @@ -345,7 +345,7 @@ analyze_ports() { statusFunc() { # Determine if there is pihole-FTL service is listening - local pid port ftl_pid_file + local pid port ftl_pid_file block_status ftl_pid_file="$(getFTLPIDFile)" @@ -375,7 +375,8 @@ statusFunc() { fi # Determine if Pi-hole's blocking is enabled - if getFTLConfigValue dns.blocking.active; then + block_status=$(getFTLConfigValue dns.blocking.active) + if [ ${block_status} == "true" ]; then case "${1}" in "web") echo "$port";; *) echo -e " ${TICK} Pi-hole blocking is enabled";; From d77dbf736cbeb1f5bc7e885e7ccb0b8cfc9c1bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 18 Oct 2023 22:09:38 +0200 Subject: [PATCH 207/462] Logrotate config file needs to be owned by root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 4 ++++ automated install/basic-install.sh | 3 ++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index abeaabc4..c6817828 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -16,6 +16,10 @@ chown -R pihole:pihole /etc/pihole /var/log/pihole chmod -R 0640 /var/log/pihole chmod -R 0660 /etc/pihole +# Logrotate config file need to be owned by root and must not be writable by group and others +chown root:root /etc/pihole/logrotate +chmod 0644 /etc/pihole/logrotate + # allow all users to enter the directories chmod 0755 /etc/pihole /var/log/pihole diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ac516415..4d656283 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1483,7 +1483,8 @@ installLogrotate() { return 2 fi # Copy the file over from the local repo - install -o pihole -g pihole -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} + # Logrotate config file must be owned by root and not writable by group or other + install -o root -g root -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. From c571d8d37d9b92336024fdf8e5fe98cfa2aee517 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 19 Oct 2023 22:04:11 +0200 Subject: [PATCH 208/462] Drop support for ancient ARMv4 and ARMv5, the Debian binaries turn out to cause unresolvable SIGFPE on ARMv6 and lower. Fortunately, we can still use the Alpine binaries on all Raspberry Pi models (even the oldest ones) Signed-off-by: DL6ER --- automated install/basic-install.sh | 26 ++--- test/test_any_automated_install.py | 154 +---------------------------- 2 files changed, 10 insertions(+), 170 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4d656283..b6806141 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1870,9 +1870,9 @@ get_binary_name() { printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" # set the binary to be used l_binary="pihole-FTL-arm64" - elif [[ "${cpu_arch}" == "armv6KZ" ]]; then - printf "%b %b Detected ARMv6KZ architecture\\n" "${OVER}" "${TICK}" - # set the binary to be used + elif [[ "${cpu_arch}" == "armv6"* ]]; then + printf "%b %b Detected ARMv6 architecture\\n" "${OVER}" "${TICK}" + # set the binary to be used (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) l_binary="pihole-FTL-armv6" else # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) @@ -1885,24 +1885,10 @@ get_binary_name() { printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" # set the binary to be used l_binary="pihole-FTL-armv6" - elif [[ "${cpu_arch}" == "v5TE" || "${rev}" -gt 5 ]]; then - # Check if the system is using GLIBC 2.29 or higher - if [[ -n "${l_glibc_version}" && "$(printf '%s\n' "2.29" "${l_glibc_version}" | sort -V | head -n1)" == "2.29" ]]; then - # If so, use the ARMv6 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 architecture (running GLIBC 2.29 or higher, %s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv5" - else - # Otherwise, use the ARMv5 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv6 architecture (running GLIBC older than 2.29, %s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv4" - fi else - # Otherwise, use the ARMv4 binary (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) - printf "%b %b Detected ARMv4 or ARMv5 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv4" + # Otherwise, Pi-hole does not support this architecture + printf "%b %b This processor architecture is not supported by Pi-hole (%s)\\n" "${OVER}" "${CROSS}" "${cpu_arch}" + l_binary="" fi fi elif [[ "${machine}" == "x86_64" ]]; then diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index a8e0e082..1980f0b6 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -259,158 +259,12 @@ def test_FTL_detect_aarch64_no_errors(host): assert expected_stdout in detectPlatform.stdout -def test_FTL_detect_armv4_no_errors(host): - """ - confirms only armv4 package is downloaded for FTL engine - """ - # mock uname to return armv4 platform - mock_command("uname", {"-m": ("armv4t", "0")}, host) - # mock readelf to respond with armv4 CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv4t", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv4t", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture (armv4t)" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv5_no_errors(host): - """ - confirms only armv5 package is downloaded for FTL engine - """ - # mock uname to return armv5te platform - mock_command("uname", {"-m": ("armv5te", "0")}, host) - # mock readelf to respond with armv5 CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv5te", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv5te", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv4 or ARMv5 architecture (armv5te)" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv6_old_no_errors(host): +def test_FTL_detect_armv6_no_errors(host): """ confirms only armv6 package is downloaded for FTL engine """ - # mock uname to return armv6l platform - mock_command("uname", {"-m": ("armv6l", "0")}, host) - # mock readelf to respond with armv6l CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv6l", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv6l", "0"), - }, - host, - ) - # Mock old ldd GLIBC version - mock_command( - "ldd", {"--version": ("ldd (Debian GLIBC 2.13-38+deb7u8) 2.13", "0")}, host - ) - - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + ( - " Detected ARMv6 architecture (running GLIBC older than 2.29, armv6l)" - ) - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv6_recent_no_errors(host): - """ - confirms only armv6 package is downloaded for FTL engine - """ - # mock uname to return armv6l platform - mock_command("uname", {"-m": ("armv6l", "0")}, host) - # mock readelf to respond with armv6l CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv6l", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv6l", "0"), - }, - host, - ) - # Mock old ldd GLIBC version - mock_command( - "ldd", {"--version": ("'ldd (Debian GLIBC 2.35-38+deb7u8) 2.35'", "0")}, host - ) - - detectPlatform = host.run( - """ - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + ( - " Detected ARMv6 architecture (running GLIBC 2.29 or higher, armv6l)" - ) - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv6KZ_no_errors(host): - """ - confirms only armv6KZ package is downloaded for FTL engine - """ - # mock uname to return armv6KZ platform - mock_command("uname", {"-m": ("armv6KZ", "0")}, host) + # mock uname to return armv6 platform + mock_command("uname", {"-m": ("armv6", "0")}, host) # mock readelf to respond with armv6l CPU architecture mock_command_2( "readelf", @@ -432,7 +286,7 @@ def test_FTL_detect_armv6KZ_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv6KZ architecture" + expected_stdout = tick_box + " Detected ARMv6 architecture" assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout From 79ebbacc4af790164ee927443bb53b55d7a87a93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 21 Oct 2023 10:27:01 +0000 Subject: [PATCH 209/462] Bump actions/checkout from 4.1.0 to 4.1.1 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.0 to 4.1.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.0...v4.1.1) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4685aa2c..6544db61 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 0ff0a24a..d9de09d2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label $stale_label env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index a1025629..e52d4ae9 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f32302e..8166d253 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.1 From 2a72012ca1448b8e2f7f0de3e79c6b9d7d72b7cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 21 Oct 2023 10:55:06 +0000 Subject: [PATCH 210/462] Bump actions/checkout from 4.1.0 to 4.1.1 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.0 to 4.1.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.0...v4.1.1) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 4685aa2c..6544db61 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 14c55d10..0e149c79 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index a1025629..e52d4ae9 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 623590da..b070c982 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Check scripts in repository are executable run: | @@ -65,7 +65,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.0 + uses: actions/checkout@v4.1.1 - name: Set up Python 3.10 uses: actions/setup-python@v4.7.1 From ca7836bf717c34f97719ba8aa3c3f6b07f0a591d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 19:05:45 +0200 Subject: [PATCH 211/462] Remove now obsolete GLIBC version check Signed-off-by: DL6ER --- automated install/basic-install.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index b9ed04a4..443ee85e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1860,15 +1860,6 @@ get_binary_name() { local machine machine=$(uname -m) - # Get local GLIBC version (leave at "0.0" if no GLIBC, e.g., on musl) - local l_glibc_version="0.0" - if ldd --version 2>&1 | grep -q "GLIBC"; then - l_glibc_version=$(ldd --version | head -n1 | grep -o '[0-9.]*$') - printf "%b %b Detected GLIBC version %s\\n" "${OVER}" "${TICK}" "${l_glibc_version}" - else - printf "%b %b No GLIBC detected\\n" "${OVER}" "${CROSS}" - fi - local l_binary local str="Detecting processor" From 45687d675becf21f23953625c00efdb637531c21 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 19:13:45 +0200 Subject: [PATCH 212/462] Fix the binary detection for ARMv6 and simplify the router overall Signed-off-by: DL6ER --- automated install/basic-install.sh | 37 ++++++++++++++---------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 443ee85e..054d8eef 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1875,29 +1875,27 @@ get_binary_name() { local rev rev=$(echo "${cpu_arch}" | grep -o '[0-9]*') if [[ "${machine}" == "aarch64" ]]; then + # If AArch64 is found (e.g., BCM2711 in Raspberry Pi 4) printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" - # set the binary to be used l_binary="pihole-FTL-arm64" - elif [[ "${cpu_arch}" == "armv6"* ]]; then + elif [[ "${cpu_arch}" == "v6"* ]]; then + # If ARMv6 is found (e.g., BCM2835 in Raspberry Pi 1 and Zero) printf "%b %b Detected ARMv6 architecture\\n" "${OVER}" "${TICK}" - # set the binary to be used (e.g., BCM2835 as found in Raspberry Pi Zero and Model 1) + l_binary="pihole-FTL-armv6" + elif [[ "${cpu_arch}" == "v7"* || "${rev}" -ge 7 ]]; then + # If ARMv7 or higher is found (e.g., BCM2836 in Raspberry PI 2 Mod. B) + # This path is also used for ARMv8 when the OS is in 32bit mode + # (e.g., BCM2837 in Raspberry Pi Model 3B, or BCM2711 in Raspberry Pi 4) + printf "%b %b Detected ARMv7 (or newer) architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" + l_binary="pihole-FTL-armv7" + elif [[ "${rev}" -gt 6 ]]; then + # Otherwise, if ARMv7 is found (e.g., BCM2836 in Raspberry Pi Model 2) + printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" l_binary="pihole-FTL-armv6" else - # If ARMv8 or higher is found (e.g., BCM2837 as found in Raspberry Pi Model 3B) - if [[ "${cpu_arch}" == "v7" || "${rev}" -gt 7 ]]; then - printf "%b %b Detected ARMv7 (or newer) architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv7" - elif [[ "${rev}" -gt 6 ]]; then - # Otherwise, if ARMv7 is found (e.g., BCM2836 as found in Raspberry Pi Model 2) - printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - # set the binary to be used - l_binary="pihole-FTL-armv6" - else - # Otherwise, Pi-hole does not support this architecture - printf "%b %b This processor architecture is not supported by Pi-hole (%s)\\n" "${OVER}" "${CROSS}" "${cpu_arch}" - l_binary="" - fi + # Otherwise, Pi-hole does not support this architecture + printf "%b %b This processor architecture is not supported by Pi-hole (%s)\\n" "${OVER}" "${CROSS}" "${cpu_arch}" + l_binary="" fi elif [[ "${machine}" == "x86_64" ]]; then # This gives the processor of packages dpkg installs (for example, "i386") @@ -1912,9 +1910,8 @@ get_binary_name() { printf "%b %b Detected 32bit (i686) architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-386" else - # 64bit + # 64bit OS printf "%b %b Detected x86_64 architecture\\n" "${OVER}" "${TICK}" - # set the binary to be used l_binary="pihole-FTL-amd64" fi elif [[ "${machine}" == "riscv64" ]]; then From 30bfc7cc9f44cd9b0138ebc2e7400f4e592fbd42 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 19:15:10 +0200 Subject: [PATCH 213/462] Add binutils tot he OS check dependencies as we need it to check if the local system is abel to run any of our precompiled FTL binaries Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 054d8eef..019cc3e3 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -302,9 +302,9 @@ package_manager_detect() { # Update package cache update_package_cache || exit 1 # Packages required to perform the os_check (stored as an array) - OS_CHECK_DEPS=(grep dnsutils) + OS_CHECK_DEPS=(grep dnsutils binutils) # Packages required to run this install script (stored as an array) - INSTALLER_DEPS=(git iproute2 dialog ca-certificates binutils) + INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole (stored as an array) PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) From 38ecc1693f50764209b8a0eff289064eef0ed881 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 19:28:59 +0200 Subject: [PATCH 214/462] Further simplify the ARMv6 test Signed-off-by: DL6ER --- automated install/basic-install.sh | 8 ++------ test/test_any_automated_install.py | 8 ++++---- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 019cc3e3..2d47a548 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1878,20 +1878,16 @@ get_binary_name() { # If AArch64 is found (e.g., BCM2711 in Raspberry Pi 4) printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-arm64" - elif [[ "${cpu_arch}" == "v6"* ]]; then + elif [[ "${rev}" -eq 6 ]]; then # If ARMv6 is found (e.g., BCM2835 in Raspberry Pi 1 and Zero) printf "%b %b Detected ARMv6 architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-armv6" - elif [[ "${cpu_arch}" == "v7"* || "${rev}" -ge 7 ]]; then + elif [[ "${rev}" -ge 7 ]]; then # If ARMv7 or higher is found (e.g., BCM2836 in Raspberry PI 2 Mod. B) # This path is also used for ARMv8 when the OS is in 32bit mode # (e.g., BCM2837 in Raspberry Pi Model 3B, or BCM2711 in Raspberry Pi 4) printf "%b %b Detected ARMv7 (or newer) architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" l_binary="pihole-FTL-armv7" - elif [[ "${rev}" -gt 6 ]]; then - # Otherwise, if ARMv7 is found (e.g., BCM2836 in Raspberry Pi Model 2) - printf "%b %b Detected ARMv7 architecture (%s)\\n" "${OVER}" "${TICK}" "${cpu_arch}" - l_binary="pihole-FTL-armv6" else # Otherwise, Pi-hole does not support this architecture printf "%b %b This processor architecture is not supported by Pi-hole (%s)\\n" "${OVER}" "${CROSS}" "${cpu_arch}" diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 1980f0b6..7d9d15f6 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -269,8 +269,8 @@ def test_FTL_detect_armv6_no_errors(host): mock_command_2( "readelf", { - "-A /bin/sh": ("Tag_CPU_arch: armv6KZ", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv6KZ", "0"), + "-A /bin/sh": ("Tag_CPU_arch: armv6", "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: armv6", "0"), }, host, ) @@ -319,7 +319,7 @@ def test_FTL_detect_armv7l_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 architecture (armv7l)") + expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout @@ -352,7 +352,7 @@ def test_FTL_detect_armv7_no_errors(host): ) expected_stdout = info_box + " FTL Checks..." assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 architecture (armv7)") + expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") assert expected_stdout in detectPlatform.stdout expected_stdout = tick_box + " Downloading and Installing FTL" assert expected_stdout in detectPlatform.stdout From 369ccf13a846bb64ac7b6d8a454f6bda4cf2a89a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 21 Oct 2023 21:40:21 +0200 Subject: [PATCH 215/462] Move FTL binary availability check after the supported OS check Signed-off-by: DL6ER --- automated install/basic-install.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 2d47a548..1ecde7a9 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2088,16 +2088,6 @@ main() { fi fi - # Check if there is a usable FTL binary available on this architecture - do - # this early on as FTL is a hard dependency for Pi-hole - local funcOutput - funcOutput=$(get_binary_name) #Store output of get_binary_name here - # Abort early if this processor is not supported (get_binary_name returns empty string) - if [[ "${funcOutput}" == "" ]]; then - printf " %b Upgrade/install aborted\\n" "${CROSS}" "${DISTRO_NAME}" - exit 1 - fi - # Check if SELinux is Enforcing and exit before doing anything else checkSelinux @@ -2114,6 +2104,16 @@ main() { # Check that the installed OS is officially supported - display warning if not os_check + # Check if there is a usable FTL binary available on this architecture - do + # this early on as FTL is a hard dependency for Pi-hole + local funcOutput + funcOutput=$(get_binary_name) #Store output of get_binary_name here + # Abort early if this processor is not supported (get_binary_name returns empty string) + if [[ "${funcOutput}" == "" ]]; then + printf " %b Upgrade/install aborted\\n" "${CROSS}" "${DISTRO_NAME}" + exit 1 + fi + # Install packages used by this installation script printf " %b Checking for / installing Required dependencies for this install script...\\n" "${INFO}" install_dependent_packages "${INSTALLER_DEPS[@]}" From 7523c49f624a57f5b0ae87e39328813bf5b72f83 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 22 Oct 2023 08:14:11 +0200 Subject: [PATCH 216/462] Swapping the databases must be the last step before the cleanup. Otherwise, FTL may be reloading from an only partially completed database causing spurious errors and/or warnings. Signed-off-by: DL6ER --- gravity.sh | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/gravity.sh b/gravity.sh index e61ade42..7b7234ee 100755 --- a/gravity.sh +++ b/gravity.sh @@ -73,9 +73,9 @@ generate_gravity_database() { chmod g+w "${piholeDir}" "${gravityDBfile}" } -# Copy data from old to new database file and swap them -gravity_swap_databases() { - local str copyGravity oldAvail +# Build gravity tree +gravity_build_tree() { + local str str="Building tree" echo -ne " ${INFO} ${str}..." @@ -88,7 +88,10 @@ gravity_swap_databases() { return 1 fi echo -e "${OVER} ${TICK} ${str}" +} +# Copy data from old to new database file and swap them +gravity_swap_databases() { str="Swapping databases" echo -ne " ${INFO} ${str}..." @@ -116,11 +119,11 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to update gravity timestamp in database ${gravityDBfile}\\n ${output}" + echo -e "\\n ${CROSS} Unable to update gravity timestamp in database ${gravityTEMPfile}\\n ${output}" return 1 fi return 0 @@ -659,12 +662,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" + unique="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi @@ -882,22 +885,27 @@ fi # Create local.list gravity_generateLocalList -# Migrate rest of the data from old to new database -if ! gravity_swap_databases; then - echo -e " ${CROSS} Unable to create database. Please contact support." - exit 1 -fi - # Update gravity timestamp update_gravity_timestamp # Ensure proper permissions are set for the database -chown pihole:pihole "${gravityDBfile}" -chmod g+w "${piholeDir}" "${gravityDBfile}" +chown pihole:pihole "${gravityTEMPfile}" +chmod g+w "${piholeDir}" "${gravityTEMPfile}" -# Compute numbers to be displayed +# Build the tree +gravity_build_tree + +# Compute numbers to be displayed (do this after building the tree to get the +# numbers quickly from the tree instead of having to scan the whole database) gravity_ShowCount +# Migrate rest of the data from old to new database +# IMPORTANT: Swapping the databases must be the last step before the cleanup +if ! gravity_swap_databases; then + echo -e " ${CROSS} Unable to create database. Please contact support." + exit 1 +fi + gravity_Cleanup echo "" From 842a9d7778e5de72f99d39b4fe78ed8523c94eb1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 22 Oct 2023 08:32:52 +0200 Subject: [PATCH 217/462] Address review comments Signed-off-by: DL6ER --- automated install/basic-install.sh | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1ecde7a9..87bf1ea4 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -90,7 +90,6 @@ IPV6_ADDRESS=${IPV6_ADDRESS} QUERY_LOGGING=true WEBPORT=8080 PRIVACY_LEVEL=0 -CACHE_SIZE=10000 if [ -z "${USER}" ]; then USER="$(id -un)" @@ -301,11 +300,11 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" # Update package cache update_package_cache || exit 1 - # Packages required to perform the os_check (stored as an array) + # Packages required to perform the os_check and FTL binary detection OS_CHECK_DEPS=(grep dnsutils binutils) - # Packages required to run this install script (stored as an array) + # Packages required to run this install script INSTALLER_DEPS=(git iproute2 dialog ca-certificates) - # Packages required to run Pi-hole (stored as an array) + # Packages required to run Pi-hole PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) # If apt-get is not found, check for rpm. @@ -1856,29 +1855,30 @@ remove_dir() { } get_binary_name() { - # Get the OS architecture (we cannot use uname -m as this may return an incorrect architecture when buildx-compiling with QEMU for arm) + local l_binary local machine machine=$(uname -m) - local l_binary - local str="Detecting processor" printf " %b %s..." "${INFO}" "${str}" - # If the machine is arm or aarch - if [[ "${machine}" == "arm"* || "${machine}" == *"aarch"* ]]; then - # ARM + + # If the machine is aarch64 (armv8) + if [[ "${machine}" == "aarch64" ]]; then + # If AArch64 is found (e.g., BCM2711 in Raspberry Pi 4) + printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" + l_binary="pihole-FTL-arm64" + elif [[ "${machine}" == "arm"* ]]; then + # ARM 32 bit # Get supported processor from other binaries installed on the system + # We cannot really rely on the output of $(uname -m) above as this may + # return an incorrect architecture when buildx-compiling with QEMU local cpu_arch cpu_arch=$(readelf -A "$(command -v sh)" | grep Tag_CPU_arch | awk '{ print $2 }') # Get the revision from the CPU architecture local rev rev=$(echo "${cpu_arch}" | grep -o '[0-9]*') - if [[ "${machine}" == "aarch64" ]]; then - # If AArch64 is found (e.g., BCM2711 in Raspberry Pi 4) - printf "%b %b Detected AArch64 (64 Bit ARM) architecture\\n" "${OVER}" "${TICK}" - l_binary="pihole-FTL-arm64" - elif [[ "${rev}" -eq 6 ]]; then + if [[ "${rev}" -eq 6 ]]; then # If ARMv6 is found (e.g., BCM2835 in Raspberry Pi 1 and Zero) printf "%b %b Detected ARMv6 architecture\\n" "${OVER}" "${TICK}" l_binary="pihole-FTL-armv6" From fd8fdd3513f7d41afddfe1877d11261c7e4ac0b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 23 Oct 2023 21:36:18 +0200 Subject: [PATCH 218/462] Use suffixed temp file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gravity.sh b/gravity.sh index 7b7234ee..edfe89a9 100755 --- a/gravity.sh +++ b/gravity.sh @@ -141,6 +141,7 @@ database_table_from_file() { # implementations of mktemp support it, e.g. on Alpine tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}")" mv "${tmpFile}" "${tmpFile%.*}.gravity" + tmpFile="${tmpFile%.*}.gravity" local timestamp timestamp="$(date --utc +'%s')" @@ -502,6 +503,7 @@ gravity_DownloadBlocklistFromUrl() { # We don't use '--suffix' here because not all implementations of mktemp support it, e.g. on Alpine listCurlBuffer="$(mktemp -p "${GRAVITY_TMPDIR}")" mv "${listCurlBuffer}" "${listCurlBuffer%.*}.phgpb" + listCurlBuffer="${listCurlBuffer%.*}.phgpb" # Determine if $saveLocation has read permission if [[ -r "${saveLocation}" && $url != "file"* ]]; then From 00d7e998b462949fa0091a4b55ad82f1572b07c4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:03:09 +0200 Subject: [PATCH 219/462] setupVars.conf and pihole-FTL.conf are no more - use pihole.toml instead Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 36 ++++++++++----------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 0e3bbf3d..977bce35 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -77,7 +77,7 @@ PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" -PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole-FTL.conf" +PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" @@ -138,7 +138,6 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}" "${PIHOLE_RAW_BLOCKLIST_FILES}" "${PIHOLE_LOCAL_HOSTS_FILE}" "${PIHOLE_LOGROTATE_FILE}" -"${PIHOLE_SETUP_VARS_FILE}" "${PIHOLE_FTL_CONF_FILE}" "${PIHOLE_COMMAND}" "${PIHOLE_COLTABLE_FILE}" @@ -165,20 +164,6 @@ show_disclaimer(){ log_write "${DISCLAIMER}" } -source_setup_variables() { - # Display the current test that is running - log_write "\\n${COL_PURPLE}*** [ INITIALIZING ]${COL_NC} Sourcing setup variables" - # If the variable file exists, - if ls "${PIHOLE_SETUP_VARS_FILE}" 1> /dev/null 2>&1; then - log_write "${INFO} Sourcing ${PIHOLE_SETUP_VARS_FILE}..."; - # source it - source ${PIHOLE_SETUP_VARS_FILE} - else - # If it can't, show an error - log_write "${PIHOLE_SETUP_VARS_FILE} ${COL_RED}does not exist or cannot be read.${COL_NC}" - fi -} - make_temporary_log() { # Create a random temporary file for the log TEMPLOG=$(mktemp /tmp/pihole_temp.XXXXXX) @@ -546,15 +531,15 @@ disk_usage() { done } -parse_setup_vars() { - echo_current_diagnostic "Setup variables" +parse_pihole_toml() { + echo_current_diagnostic "Pi-hole configuration" # If the file exists, - if [[ -r "${PIHOLE_SETUP_VARS_FILE}" ]]; then + if [[ -r "${PIHOLE_FTL_CONF_FILE}" ]]; then # parse it - parse_file "${PIHOLE_SETUP_VARS_FILE}" + parse_file "${PIHOLE_FTL_CONF_FILE}" else # If not, show an error - log_write "${CROSS} ${COL_RED}Could not read ${PIHOLE_SETUP_VARS_FILE}.${COL_NC}" + log_write "${CROSS} ${COL_RED}Could not read ${PIHOLE_FTL_CONF_FILE}.${COL_NC}" fi } @@ -1010,8 +995,10 @@ parse_file() { # For each line in the file, for file_lines in "${file_info[@]}"; do if [[ -n "${file_lines}" ]]; then - # don't include the Web password hash - [[ "${file_lines}" =~ ^\#.*$ || ! "${file_lines}" || "${file_lines}" == "WEBPASSWORD="* ]] && continue + # skip empty and comment lines line + [[ "${file_lines}" =~ ^[[:space:]]*\#.*$ || ! "${file_lines}" ]] && continue + # remove the password hash from the output (*"pwhash = "*) + [[ "${file_lines}" == *"pwhash ="* ]] && file_lines=$(echo "${file_lines}" | sed -e 's/\(pwhash = \).*/\1/') # otherwise, display the lines of the file log_write " ${file_lines}" fi @@ -1076,7 +1063,6 @@ list_files_in_dir() { elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_RAW_BLOCKLIST_FILES}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_INSTALL_LOG_FILE}" ]] || \ - [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_SETUP_VARS_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG_GZIPS}" ]]; then @@ -1484,7 +1470,7 @@ check_dhcp_servers process_status ftl_full_status lighttpd_test_configuration -parse_setup_vars +parse_pihole_toml check_x_headers analyze_ftl_db analyze_gravity_list From ccd8c470a535c5a42085f1e68a91d4b4bff5bf18 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:03:37 +0200 Subject: [PATCH 220/462] check_x_headers is no more Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 977bce35..88723cdc 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -732,35 +732,6 @@ check_networking() { [ -z "${DOCKER_VERSION}" ] && check_required_ports } -# check_x_headers() { -# # The X-Headers allow us to determine from the command line if the Web -# # lighttpd.conf has a directive to show "X-Pi-hole: A black hole for Internet advertisements." -# # in the header of any Pi-holed domain -# # Similarly, it will show "X-Pi-hole: The Pi-hole Web interface is working!" if you view the header returned -# # when accessing the dashboard (i.e curl -I pi.hole/admin/) -# # server is operating correctly -# echo_current_diagnostic "Dashboard headers" -# # Use curl -I to get the header and parse out just the X-Pi-hole one -# local full_curl_output_dashboard -# local dashboard -# full_curl_output_dashboard="$(curl -Is localhost/admin/)" -# dashboard=$(echo "${full_curl_output_dashboard}" | awk '/X-Pi-hole/' | tr -d '\r') -# # Store what the X-Header should be in variables for comparison later -# local dashboard_working -# dashboard_working="X-Pi-hole: The Pi-hole Web interface is working!" - -# # If the X-Header matches what a working system should have, -# if [[ $dashboard == "$dashboard_working" ]]; then -# # then we can show a success -# log_write "$TICK Web interface X-Header: ${COL_GREEN}${dashboard}${COL_NC}" -# else -# # Otherwise, it's a failure since the X-Headers either don't exist or have been modified in some way -# log_write "$CROSS Web interface X-Header: ${COL_RED}X-Header does not match or could not be retrieved.${COL_NC}" - -# log_write "${COL_RED}${full_curl_output_dashboard}${COL_NC}" -# fi -# } - dig_at() { # We need to test if Pi-hole can properly resolve domain names # as it is an essential piece of the software @@ -1471,7 +1442,6 @@ process_status ftl_full_status lighttpd_test_configuration parse_pihole_toml -check_x_headers analyze_ftl_db analyze_gravity_list show_groups From dcc7e96d2f4b963cae8630a6db3e04b7fc7d2802 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:07:11 +0200 Subject: [PATCH 221/462] Remove detection of addresses of PIHOLE_INTERFACE as it is gone, too Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 35 +++------------------------------ 1 file changed, 3 insertions(+), 32 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 88723cdc..517ab4ec 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -550,33 +550,6 @@ parse_locale() { parse_file "${pihole_locale}" } -detect_ip_addresses() { - # First argument should be a 4 or a 6 - local protocol=${1} - # Use ip to show the addresses for the chosen protocol - # Store the values in an array so they can be looped through - # Get the lines that are in the file(s) and store them in an array for parsing later - mapfile -t ip_addr_list < <(ip -"${protocol}" addr show dev "${PIHOLE_INTERFACE}" | awk -F ' ' '{ for(i=1;i<=NF;i++) if ($i ~ '/^inet/') print $(i+1) }') - - # If there is something in the IP address list, - if [[ -n ${ip_addr_list[*]} ]]; then - # Local iterator - local i - # Display the protocol and interface - log_write "${TICK} IPv${protocol} address(es) bound to the ${PIHOLE_INTERFACE} interface:" - # Since there may be more than one IP address, store them in an array - for i in "${!ip_addr_list[@]}"; do - log_write " ${ip_addr_list[$i]}" - done - # Print a blank line just for formatting - log_write "" - else - # If there are no IPs detected, explain that the protocol is not configured - log_write "${CROSS} ${COL_RED}No IPv${protocol} address(es) found on the ${PIHOLE_INTERFACE}${COL_NC} interface.\\n" - return 1 - fi -} - ping_ipv4_or_ipv6() { # Give the first argument a readable name (a 4 or a six should be the argument) local protocol="${1}" @@ -605,9 +578,9 @@ ping_gateway() { while IFS= read -r gateway; do log_write " ${gateway}" - done < <(ip -"${protocol}" route | grep default | grep "${PIHOLE_INTERFACE}" | cut -d ' ' -f 3) + done < <(ip -"${protocol}" route | grep default | cut -d ' ' -f 3) - gateway=$(ip -"${protocol}" route | grep default | grep "${PIHOLE_INTERFACE}" | cut -d ' ' -f 3 | head -n 1) + gateway=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 3 | head -n 1) # If there was at least one gateway if [ -n "${gateway}" ]; then # Let the user know we will ping the gateway for a response @@ -615,7 +588,7 @@ ping_gateway() { # Try to quietly ping the gateway 3 times, with a timeout of 3 seconds, using numeric output only, # on the pihole interface, and tail the last three lines of the output # If pinging the gateway is not successful, - if ! ${cmd} -c 1 -W 2 -n "${gateway}" -I "${PIHOLE_INTERFACE}" >/dev/null; then + if ! ${cmd} -c 1 -W 2 -n "${gateway}" >/dev/null; then # let the user know log_write "${CROSS} ${COL_RED}Gateway did not respond.${COL_NC} ($FAQ_GATEWAY)\\n" # and return an error code @@ -723,8 +696,6 @@ check_networking() { # Runs through several of the functions made earlier; we just clump them # together since they are all related to the networking aspect of things echo_current_diagnostic "Networking" - detect_ip_addresses "4" - detect_ip_addresses "6" ping_gateway "4" ping_gateway "6" # Skip the following check if installed in docker container. Unpriv'ed containers do not have access to the information required From c86e7b3f96ebcd3c99b19a360f2b7410a6847778 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:13:31 +0200 Subject: [PATCH 222/462] Generalize CPU check to a short hardware information output Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 50 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 517ab4ec..4c0e3cd0 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -482,34 +482,34 @@ check_firewalld() { fi } -processor_check() { - echo_current_diagnostic "Processor" - # Store the processor type in a variable - PROCESSOR=$(uname -m) - # If it does not contain a value, - if [[ -z "${PROCESSOR}" ]]; then - # we couldn't detect it, so show an error - PROCESSOR=$(lscpu | awk '/Architecture/ {print $2}') - log_write "${CROSS} ${COL_RED}${PROCESSOR}${COL_NC} has not been tested with FTL, but may still work: (${FAQ_FTL_COMPATIBILITY})" +run_and_print_command() { + # Run the command passed as an argument + local cmd="${1}" + # Show the command that is being run + log_write "${INFO} ${cmd}" + # Run the command and store the output in a variable + local output + output=$(${cmd} 2>&1) + # If the command was successful, + if [[ $? -eq 0 ]]; then + # show the output + log_write "${output}" else - # Check if the architecture is currently supported for FTL - case "${PROCESSOR}" in - "amd64" | "x86_64") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - "armv6l") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - "armv6") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - "armv7l") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - "aarch64") log_write "${TICK} ${COL_GREEN}${PROCESSOR}${COL_NC}" - ;; - # Otherwise, show the processor type - *) log_write "${INFO} ${PROCESSOR}"; - esac + # otherwise, show an error + log_write "${CROSS} ${COL_RED}Command failed${COL_NC}" fi } +hardware_check() { + echo_current_diagnostic "System hardware configuration" + # Store the output of the command in a variable + run_and_print_command "lshw -short" + + echo_current_diagnostic "Processor details" + # Store the output of the command in a variable + run_and_print_command "lscpu" +} + disk_usage() { local file_system local hide @@ -1403,7 +1403,7 @@ check_component_versions diagnose_operating_system check_selinux check_firewalld -processor_check +hardware_check disk_usage check_ip_command check_networking From 635b4e952f9c18c1f3ba20280f17b1c1eab20463 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:20:31 +0200 Subject: [PATCH 223/462] Add NOERROR/NXDOMAIN support in the DNS resolution test Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 4c0e3cd0..9225c059 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -781,8 +781,16 @@ dig_at() { if [ -n "${addresses}" ]; then while IFS= read -r local_address ; do # Check if Pi-hole can use itself to block a domain - if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" +short "${record_type}"); then + if local_dig="$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" "${record_type}")"; then # If it can, show success + if [[ "${local_dig}" == *"status: NOERROR"* ]]; then + local_dig="NOERROR" + elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then + local_dig="NXDOMAIN" + else + # Extract the IPv4/6 address from the output + local_dig="$(echo "${local_dig}" | grep -Eo '[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*|([0-9a-f]{0,4}:){1,7}[0-9a-f]{0,4}')" + fi log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" else # Otherwise, show a failure From 15be8eeffad5fd209ff08ca8078bd43a4989274c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:21:08 +0200 Subject: [PATCH 224/462] Remove lighttpd config test Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 9225c059..0e57722e 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -868,20 +868,6 @@ ftl_full_status(){ fi } -lighttpd_test_configuration(){ - # let lighttpd test it's own configuration - local lighttpd_conf_test - echo_current_diagnostic "Lighttpd configuration test" - lighttpd_conf_test=$(lighttpd -tt -f /etc/lighttpd/lighttpd.conf) - if [ -z "${lighttpd_conf_test}" ]; then - # empty output - log_write "${TICK} ${COL_GREEN}No error in lighttpd configuration${COL_NC}" - else - log_write "${CROSS} ${COL_RED}Error in lighttpd configuration${COL_NC}" - log_write " ${lighttpd_conf_test}" - fi -} - make_array_from_file() { local filename="${1}" # The second argument can put a limit on how many line should be read from the file @@ -1419,7 +1405,6 @@ check_name_resolution check_dhcp_servers process_status ftl_full_status -lighttpd_test_configuration parse_pihole_toml analyze_ftl_db analyze_gravity_list From d377cfbc3eb745f112f741f8997d57191f4daa37 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 28 Oct 2023 08:34:30 +0200 Subject: [PATCH 225/462] Extract the first entry in the answer section from dig's output, replacing any multiple spaces and tabs with a single space Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 0e57722e..53123b9a 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -788,8 +788,9 @@ dig_at() { elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then local_dig="NXDOMAIN" else - # Extract the IPv4/6 address from the output - local_dig="$(echo "${local_dig}" | grep -Eo '[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*|([0-9a-f]{0,4}:){1,7}[0-9a-f]{0,4}')" + # Extract the first entry in the answer section from dig's output, + # replacing any multiple spaces and tabs with a single space + local_dig="$(echo "${local_dig}" | grep -A1 "ANSWER SECTION" | grep -v "ANSWER SECTION" | tr -s " \t" " ")" fi log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" else From 2e9e579bba9122febfa8605452455dc5397f68d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 28 Oct 2023 10:31:54 +0000 Subject: [PATCH 226/462] Bump pytest from 7.4.2 to 7.4.3 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.2 to 7.4.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.2...7.4.3) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 27417754..45c7c7c0 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.2 +pytest == 7.4.3 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 tox == 4.11.3 From ec86124997d1b8969be2f528fc63b2626486eb73 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 28 Oct 2023 10:32:55 +0000 Subject: [PATCH 227/462] Bump pytest from 7.4.2 to 7.4.3 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.2 to 7.4.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.2...7.4.3) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 27417754..45c7c7c0 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.2 +pytest == 7.4.3 pytest-xdist == 3.3.1 pytest-testinfra == 9.0.0 tox == 4.11.3 From 6292e65b376ce1d5ad350da2a0b951fc1480f744 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 28 Oct 2023 17:56:37 +0100 Subject: [PATCH 228/462] When setting a blank password, use `webserver.api.password` instead of `webserver.api.pwhash` (fixed in https://github.com/pi-hole/FTL/pull/1702) This prevents the password from being blanked out on the command line when it has been set by an environment variable --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index 9cc1069f..8e3f1f98 100755 --- a/pihole +++ b/pihole @@ -43,7 +43,7 @@ SetWebPassword() { echo "" if [ "${PASSWORD}" == "" ]; then - setFTLConfigValue "webserver.api.pwhash" "" >/dev/null + setFTLConfigValue "webserver.api.password" "" >/dev/null echo -e " ${TICK} Password Removed" exit 0 fi From ab3b6dfa67b0699a0fa895f2318a7a28f78ddca7 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 28 Oct 2023 19:46:45 +0100 Subject: [PATCH 229/462] No need to >/dev/null because setFTLConfigValue already does this adjust output of setFTLConfigValue to test for the exit code of `pihole-FTL --config` --- advanced/Scripts/utils.sh | 4 ++++ pihole | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 3f9b7031..2fe419e8 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -145,4 +145,8 @@ getFTLConfigValue(){ ####################### setFTLConfigValue(){ pihole-FTL --config "${1}" "${2}" >/dev/null + if [[ $? -eq 5 ]]; then + echo -e " ${CROSS} ${1} set by environment variable. Please unset it to use this function" + exit 5 + fi } diff --git a/pihole b/pihole index 8e3f1f98..279977e4 100755 --- a/pihole +++ b/pihole @@ -43,7 +43,7 @@ SetWebPassword() { echo "" if [ "${PASSWORD}" == "" ]; then - setFTLConfigValue "webserver.api.password" "" >/dev/null + setFTLConfigValue "webserver.api.password" "" echo -e " ${TICK} Password Removed" exit 0 fi @@ -54,7 +54,7 @@ SetWebPassword() { if [ "${PASSWORD}" == "${CONFIRM}" ] ; then # pihole-FTL will automatically hash the password - setFTLConfigValue "webserver.api.password" "${PASSWORD}" >/dev/null + setFTLConfigValue "webserver.api.password" "${PASSWORD}" echo -e " ${TICK} New password set" else echo -e " ${CROSS} Passwords don't match. Your password has not been changed" From 480a8c8d7fd588e3cb35081b6064e60c18643d4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 31 Oct 2023 21:02:26 +0100 Subject: [PATCH 230/462] Remove idn2 as punycode conversion is handled by FTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/query.sh | 10 ++++------ automated install/basic-install.sh | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 27a33f39..2279df85 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -108,13 +108,11 @@ Main(){ if [ -z "${domain}" ]; then echo "No domain specified"; exit 1 - else - # convert domain to punycode - domain=$(idn2 "${domain}") - - # convert the domain to lowercase - domain=$(echo "${domain}" | tr '[:upper:]' '[:lower:]') fi + # domains are lowercased and converted to punycode by FTL since + # https://github.com/pi-hole/FTL/pull/1715 + # no need to do it here + # Test if the authentication endpoint is available TestAPIAvailability diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 87bf1ea4..ef052dbd 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -305,7 +305,7 @@ package_manager_detect() { # Packages required to run this install script INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip idn2 libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) + PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) # If apt-get is not found, check for rpm. elif is_command rpm ; then @@ -322,7 +322,7 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) - PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc libcap nmap-ncat jq) + PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq) # If neither apt-get or yum/dnf package managers were found else From ced1c5d50b5e5beedefed9f2747a21b82b60e833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 17 Dec 2022 16:53:01 +0100 Subject: [PATCH 231/462] Start counting at postion 1 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/updatecheck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 8bb1888b..66f1a7ab 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -31,7 +31,7 @@ function get_remote_version() { function get_remote_hash(){ - git ls-remote "https://github.com/pi-hole/${1}" --tags "${2}" | awk '{print substr($0, 0,8);}' || return 1 + git ls-remote "https://github.com/pi-hole/${1}" --tags "${2}" | awk '{print substr($0, 1,8);}' || return 1 } # Source the utils file for addOrEditKeyValPair() From bc96d3b0a9635202f56ef42be6ea1ef100a5d27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 5 Nov 2023 21:15:13 +0100 Subject: [PATCH 232/462] Tweak help text of pihole setpassword MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- pihole | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pihole b/pihole index 279977e4..47da4ddd 100755 --- a/pihole +++ b/pihole @@ -496,7 +496,9 @@ Debugging Options: Options: - setpassword set the password for the web interface + setpassword [pwd] Set the password for the web interface + Without optional argument, password is read interactively. + When specifying a password directly, enclose it in single quotes. -g, updateGravity Update the list of ad-serving domains -h, --help, help Show this help dialog -l, logging Specify whether the Pi-hole log should be used From 72c972175dcb485fefc2de424c4df6eeea0c4d14 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 6 Nov 2023 13:53:08 +0100 Subject: [PATCH 233/462] Remove left-over parts of setupVars processing and only include pihole.toml once Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c53eada4..8c2388d8 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -544,18 +544,6 @@ disk_usage() { done } -parse_pihole_toml() { - echo_current_diagnostic "Pi-hole configuration" - # If the file exists, - if [[ -r "${PIHOLE_FTL_CONF_FILE}" ]]; then - # parse it - parse_file "${PIHOLE_FTL_CONF_FILE}" - else - # If not, show an error - log_write "${CROSS} ${COL_RED}Could not read ${PIHOLE_FTL_CONF_FILE}.${COL_NC}" - fi -} - parse_locale() { local pihole_locale echo_current_diagnostic "Locale" @@ -1402,10 +1390,6 @@ upload_to_tricorder() { # Run through all the functions we made make_temporary_log initialize_debug -# TODO: Address the reliance on setupVars.conf here. Should debug read pihole.toml directly, or rely on pihole-FTL --config? -# setupVars.conf needs to be sourced before the networking so the values are -# available to the other functions -source_setup_variables check_component_versions # check_critical_program_versions diagnose_operating_system @@ -1419,7 +1403,6 @@ check_name_resolution check_dhcp_servers process_status ftl_full_status -parse_pihole_toml analyze_ftl_db analyze_gravity_list show_groups From 0b4131189d1eb83974967d818334838b41b8be9e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 6 Nov 2023 14:55:34 +0100 Subject: [PATCH 234/462] Migrate Pi-hole created files out of /etc/dnsmasq.d into a pihole owned directory Signed-off-by: DL6ER --- automated install/basic-install.sh | 34 ++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef5add0e..704a736d 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -91,6 +91,9 @@ QUERY_LOGGING=true WEBPORT=8080 PRIVACY_LEVEL=0 +# Where old configs go to if a v6 migration is performed +V6_CONF_MIGRATION_DIR="/etc/pihole/migration_backup_v6" + if [ -z "${USER}" ]; then USER="$(id -un)" fi @@ -2070,6 +2073,34 @@ copy_to_install_log() { chown pihole:pihole "${installLogLoc}" } +migrate_dnsmasq_configs() { + # Previously, Pi-hole created a number of files in /etc/dnsmasq.d + # During migration, their content is copied into the new single source of + # truth file /etc/pihole/pihole.toml and the old files are moved away to + # avoid conflicts with other services on this system + + # Exit early if this is already Pi-hole v6.0 + # We decide this on the presence of the file /etc/pihole/pihole.toml + if [[ -f /etc/pihole/pihole.toml ]]; then + return 0 + fi + + # Create target directory /etc/pihole/migration_backup_v6 + # and make it owned by pihole:pihole + mkdir -p "${V6_CONF_MIGRATION_DIR} + chown pihole:pihole "${V6_CONF_MIGRATION_DIR} + + # Move all conf files originally created by Pi-hole into this directory + # - 01-pihole.conf + # - 02-pihole-dhcp.conf + # - 04-pihole-static-dhcp.conf + # - 05-pihole-custom-cname.conf + # - 06-rfc6761.conf + + mv /etc/dnsmasq.d/0{1,2,4,5}-pihole*.conf "${V6_MIGRATION_DIR}/ CONF_2>/dev/null || true + mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_MIGRATION_DIR}/ CONF_2>/dev/null || true +} + main() { ######## FIRST CHECK ######## # Must be root to install @@ -2219,6 +2250,9 @@ main() { pihole -a -p "${pw}" fi + # Migrate existing install to v6.0 + migrate_dnsmasq_configs + # Check for and disable systemd-resolved-DNSStubListener before reloading resolved # DNSStubListener needs to remain in place for installer to download needed files, # so this change needs to be made after installation is complete, From de6e61705e7d93d9c0f07ba74f343f43220770de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 6 Nov 2023 21:40:32 +0100 Subject: [PATCH 235/462] Remove local.list and openVPN traces MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 1 - gravity.sh | 25 +------------------------ 2 files changed, 1 insertion(+), 25 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 8c2388d8..55659690 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -75,7 +75,6 @@ PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole" PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" -PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" diff --git a/gravity.sh b/gravity.sh index edfe89a9..5c944454 100755 --- a/gravity.sh +++ b/gravity.sh @@ -36,9 +36,6 @@ blacklistFile="${piholeDir}/blacklist.txt" regexFile="${piholeDir}/regex.list" adListFile="${piholeDir}/adlists.list" -localList="${piholeDir}/local.list" -VPNList="/etc/openvpn/ipp.txt" - piholeGitDir="/etc/.pihole" GRAVITYDB=$(getFTLConfigValue files.gravity) gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" @@ -299,12 +296,7 @@ migrate_to_database() { # Determine if DNS resolution is available before proceeding gravity_CheckDNSResolutionAvailable() { - local lookupDomain="pi.hole" - - # Determine if $localList does not exist, and ensure it is not empty - if [[ ! -e "${localList}" ]] || [[ -s "${localList}" ]]; then - lookupDomain="raw.githubusercontent.com" - fi + local lookupDomain="raw.githubusercontent.com" # Determine if $lookupDomain is resolvable if timeout 4 getent hosts "${lookupDomain}" &> /dev/null; then @@ -686,18 +678,6 @@ gravity_ShowCount() { gravity_Table_Count "vw_regex_whitelist" "regex allowed filters" } -# Create "localhost" entries into hosts format -gravity_generateLocalList() { - # Empty $localList if it already exists, otherwise, create it - echo "### Do not modify this file, it will be overwritten by pihole -g" > "${localList}" - chmod 644 "${localList}" - - # Add additional LAN hosts provided by OpenVPN (if available) - if [[ -f "${VPNList}" ]]; then - awk -F, '{printf $2"\t"$1".vpn\n"}' "${VPNList}" >> "${localList}" - fi -} - # Trap Ctrl-C gravity_Trap() { trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT @@ -884,9 +864,6 @@ if ! gravity_DownloadBlocklists; then exit 1 fi -# Create local.list -gravity_generateLocalList - # Update gravity timestamp update_gravity_timestamp From e3c6f162d84a3cf78e6e50969424b1e6cebe742f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 7 Nov 2023 08:57:56 +0100 Subject: [PATCH 236/462] Apply suggestions from code review Co-authored-by: Adam Warner Co-authored-by: RD WebDesign Signed-off-by: DL6ER --- automated install/basic-install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 704a736d..e743a071 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2087,8 +2087,8 @@ migrate_dnsmasq_configs() { # Create target directory /etc/pihole/migration_backup_v6 # and make it owned by pihole:pihole - mkdir -p "${V6_CONF_MIGRATION_DIR} - chown pihole:pihole "${V6_CONF_MIGRATION_DIR} + mkdir -p "${V6_CONF_MIGRATION_DIR}" + chown pihole:pihole "${V6_CONF_MIGRATION_DIR}" # Move all conf files originally created by Pi-hole into this directory # - 01-pihole.conf @@ -2097,8 +2097,8 @@ migrate_dnsmasq_configs() { # - 05-pihole-custom-cname.conf # - 06-rfc6761.conf - mv /etc/dnsmasq.d/0{1,2,4,5}-pihole*.conf "${V6_MIGRATION_DIR}/ CONF_2>/dev/null || true - mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_MIGRATION_DIR}/ CONF_2>/dev/null || true + mv /etc/dnsmasq.d/0{1,2,4,5}-pihole*.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true + mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_MIGRATION_DIR}/" 2>/dev/null || true } main() { From e8884083ef1f10229629e80021107f807a7fd584 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 7 Nov 2023 08:58:31 +0100 Subject: [PATCH 237/462] Apply suggestions from code review Signed-off-by: DL6ER --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e743a071..da0133b1 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2098,7 +2098,7 @@ migrate_dnsmasq_configs() { # - 06-rfc6761.conf mv /etc/dnsmasq.d/0{1,2,4,5}-pihole*.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true - mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_MIGRATION_DIR}/" 2>/dev/null || true + mv /etc/dnsmasq.d/06-rfc6761.conf "${V6_CONF_MIGRATION_DIR}/" 2>/dev/null || true } main() { From 766cb26af5884281f343b2cba87cf7cafc207447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 7 Nov 2023 22:24:34 +0100 Subject: [PATCH 238/462] Drop Fedora 36 and add Fedora 39 to the test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 2 +- test/{_fedora_36.Dockerfile => _fedora_39.Dockerfile} | 2 +- test/{tox.fedora_36.ini => tox.fedora_39.ini} | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename test/{_fedora_36.Dockerfile => _fedora_39.Dockerfile} (97%) rename test/{tox.fedora_36.ini => tox.fedora_39.ini} (80%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8166d253..9d90b2e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,9 +64,9 @@ jobs: ubuntu_23, centos_8, centos_9, - fedora_36, fedora_37, fedora_38, + fedora_39, ] env: DISTRO: ${{matrix.distro}} diff --git a/test/_fedora_36.Dockerfile b/test/_fedora_39.Dockerfile similarity index 97% rename from test/_fedora_36.Dockerfile rename to test/_fedora_39.Dockerfile index 847767e7..1727a3aa 100644 --- a/test/_fedora_36.Dockerfile +++ b/test/_fedora_39.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:36 +FROM fedora:39 RUN dnf install -y git initscripts ENV GITDIR /etc/.pihole diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_39.ini similarity index 80% rename from test/tox.fedora_36.ini rename to test/tox.fedora_39.ini index 1896a45f..5c8557c9 100644 --- a/test/tox.fedora_36.ini +++ b/test/tox.fedora_39.ini @@ -1,8 +1,8 @@ [tox] envlist = py3 -[testenv:py3] +[testenv] allowlist_externals = docker deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_39.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From f65b12643345c74d27c1c82a7bf3c177e609cce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 13 Nov 2023 21:12:28 +0100 Subject: [PATCH 239/462] Move custom.list to /hosts/custom.list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 2 +- automated install/basic-install.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 8c2388d8..c346da57 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -78,7 +78,7 @@ PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" -PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/custom.list" +PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/hosts/custom.list" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef5add0e..9b619cbf 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1122,9 +1122,9 @@ installConfigs() { chown pihole:pihole "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" # Install empty custom.list file if it does not exist - if [[ ! -r "${PI_HOLE_CONFIG_DIR}/custom.list" ]]; then - if ! install -o pihole -g pihole -m 660 /dev/null "${PI_HOLE_CONFIG_DIR}/custom.list" &>/dev/null; then - printf " %b Error: Unable to initialize configuration file %s/custom.list\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}" + if [[ ! -r "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" ]]; then + if ! install -D -T -o pihole -g pihole -m 660 /dev/null "${PI_HOLE_CONFIG_DIR}/hosts/custom.list" &>/dev/null; then + printf " %b Error: Unable to initialize configuration file %s/custom.list\\n" "${COL_LIGHT_RED}" "${PI_HOLE_CONFIG_DIR}/hosts" return 1 fi fi From 86aa6b1df64ad263cdc6b48f3d9ce870cf0a5fe4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 6 Nov 2023 22:38:01 +0100 Subject: [PATCH 240/462] Improve v6 debug log and remove leftovers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 43 +++++++++------------------------ 1 file changed, 12 insertions(+), 31 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c346da57..a45e52a9 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -49,7 +49,6 @@ FAQ_HARDWARE_REQUIREMENTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisite FAQ_HARDWARE_REQUIREMENTS_PORTS="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/#ports${COL_NC}" FAQ_HARDWARE_REQUIREMENTS_FIREWALLD="${COL_CYAN}https://docs.pi-hole.net/main/prerequisites/#firewalld${COL_NC}" FAQ_GATEWAY="${COL_CYAN}https://discourse.pi-hole.net/t/why-is-a-default-gateway-important-for-pi-hole/3546${COL_NC}" -FAQ_FTL_COMPATIBILITY="${COL_CYAN}https://github.com/pi-hole/FTL#compatibility-list${COL_NC}" # Other URLs we may use FORUMS_URL="${COL_CYAN}https://discourse.pi-hole.net${COL_NC}" @@ -78,7 +77,6 @@ PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" -PIHOLE_CUSTOM_HOSTS_FILE="${PIHOLE_DIRECTORY}/hosts/custom.list" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. @@ -91,12 +89,12 @@ get_ftl_conf_value() { local default=$2 local value - # Obtain key=... setting from pihole-FTL.conf + # Obtain key=... setting from FTL directly if [[ -e "$PIHOLE_FTL_CONF_FILE" ]]; then # Constructed to return nothing when # a) the setting is not present in the config file, or # b) the setting is commented out (e.g. "#DBFILE=...") - value="$(sed -n -e "s/^\\s*$key=\\s*//p" ${PIHOLE_FTL_CONF_FILE})" + value="$(pihole-FTL --config "${key}")" fi # Test for missing value. Use default value in this case. @@ -107,9 +105,9 @@ get_ftl_conf_value() { echo "$value" } -PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "GRAVITYDB" "${PIHOLE_DIRECTORY}/gravity.db")" +PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity" "${PIHOLE_DIRECTORY}/gravity.db")" -PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "DBFILE" "${PIHOLE_DIRECTORY}/pihole-FTL.db")" +PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database" "${PIHOLE_DIRECTORY}/pihole-FTL.db")" PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole" PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE" @@ -119,10 +117,8 @@ FTL_PID="${RUN_DIRECTORY}/pihole-FTL.pid" PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log" PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*" PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log" -PIHOLE_FTL_LOG="$(get_ftl_conf_value "LOGFILE" "${LOG_DIRECTORY}/FTL.log")" - -# PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access-pihole.log" #TODO: FTL Error log? -# PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error-pihole.log" #TODO: FTL Error log? +PIHOLE_FTL_LOG="$(get_ftl_conf_value "files.log.ftl" "${LOG_DIRECTORY}/FTL.log")" +PIHOLE_WEBSERVER_LOG="$(get_ftl_conf_value "files.log.webserver" "${LOG_DIRECTORY}/webserver.log")" RESOLVCONF="${ETC}/resolv.conf" DNSMASQ_CONF="${ETC}/dnsmasq.conf" @@ -132,8 +128,6 @@ PIHOLE_PROCESSES=( "pihole-FTL" ) # Store the required directories in an array so it can be parsed through REQUIRED_FILES=("${PIHOLE_CRON_FILE}" -# "${WEB_SERVER_CONFIG_FILE}" -# "${WEB_SERVER_CUSTOM_CONFIG_FILE}" "${PIHOLE_INSTALL_LOG_FILE}" "${PIHOLE_RAW_BLOCKLIST_FILES}" "${PIHOLE_LOCAL_HOSTS_FILE}" @@ -146,11 +140,9 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}" "${PIHOLE_LOG_GZIPS}" "${PIHOLE_DEBUG_LOG}" "${PIHOLE_FTL_LOG}" -"${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" -"${PIHOLE_WEB_SERVER_ERROR_LOG_FILE}" +"${PIHOLE_WEBSERVER_LOG}" "${RESOLVCONF}" "${DNSMASQ_CONF}" -"${PIHOLE_CUSTOM_HOSTS_FILE}" "${PIHOLE_VERSIONS_FILE}") DISCLAIMER="This process collects information from your Pi-hole, and optionally uploads it to a unique and random directory on tricorder.pi-hole.net. @@ -983,12 +975,6 @@ list_files_in_dir() { if [[ "${dir_to_parse}" == "${SHM_DIRECTORY}" ]]; then # SHM file - we do not want to see the content, but we want to see the files and their sizes log_write "$(ls -lh "${dir_to_parse}/")" - elif [[ "${dir_to_parse}" == "${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}" ]]; then - # we want to see all files files in /etc/lighttpd/conf.d - log_write "$(ls -lh "${dir_to_parse}/" 2> /dev/null )" - elif [[ "${dir_to_parse}" == "${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}" ]]; then - # we want to see all files files in /etc/lighttpd/conf.d - log_write "$(ls -lh "${dir_to_parse}/"/ 2> /dev/null )" fi # Store the files found in an array @@ -1002,7 +988,6 @@ list_files_in_dir() { [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_RAW_BLOCKLIST_FILES}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_INSTALL_LOG_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG}" ]] || \ - [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG_GZIPS}" ]]; then : elif [[ "${dir_to_parse}" == "${DNSMASQ_D_DIRECTORY}" ]]; then @@ -1017,8 +1002,8 @@ list_files_in_dir() { log_write "\\n${COL_GREEN}$(ls -lhd "${dir_to_parse}"/"${each_file}")${COL_NC}" # Check if the file we want to view has a limit (because sometimes we just need a little bit of info from the file, not the entire thing) case "${dir_to_parse}/${each_file}" in - # If it's Web server error log, give the first and last 25 lines - "${PIHOLE_WEB_SERVER_ERROR_LOG_FILE}") head_tail_log "${dir_to_parse}/${each_file}" 25 + # If it's Web server log, give the first and last 25 lines + "${PIHOLE_WEBSERVER_LOG}") head_tail_log "${dir_to_parse}/${each_file}" 25 ;; # Same for the FTL log "${PIHOLE_FTL_LOG}") head_tail_log "${dir_to_parse}/${each_file}" 35 @@ -1049,11 +1034,7 @@ show_content_of_pihole_files() { # Show the content of the files in each of Pi-hole's folders show_content_of_files_in_dir "${PIHOLE_DIRECTORY}" show_content_of_files_in_dir "${DNSMASQ_D_DIRECTORY}" - show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY}" - show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY_FEDORA}" - show_content_of_files_in_dir "${WEB_SERVER_CONFIG_DIRECTORY_DEBIAN}" show_content_of_files_in_dir "${CRON_D_DIRECTORY}" - show_content_of_files_in_dir "${WEB_SERVER_LOG_DIRECTORY}" show_content_of_files_in_dir "${LOG_DIRECTORY}" show_content_of_files_in_dir "${SHM_DIRECTORY}" show_content_of_files_in_dir "${ETC}" @@ -1290,10 +1271,10 @@ spinner(){ analyze_pihole_log() { echo_current_diagnostic "Pi-hole log" local pihole_log_permissions - local logging_enabled + local queryLogging - logging_enabled=$(grep -c "^log-queries" /etc/dnsmasq.d/01-pihole.conf) - if [[ "${logging_enabled}" == "0" ]]; then + queryLogging=$(pihole-FTL --config dns.queryLogging) + if [[ "${queryLogging}" == "false" ]]; then # Inform user that logging has been disabled and pihole.log does not contain queries log_write "${INFO} Query logging is disabled" log_write "" From d854eb1a9789a99a24b3ca7b7c101418d47a383c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 7 Nov 2023 21:59:35 +0100 Subject: [PATCH 241/462] Add lshw to Pi-hole deps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9b619cbf..72f00992 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -332,7 +332,7 @@ package_manager_detect() { # Packages required to run this install script INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq) + PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq lshw) # If apt-get is not found, check for rpm. elif is_command rpm ; then @@ -349,7 +349,7 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) - PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq) + PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq lshw) # If neither apt-get or yum/dnf package managers were found else From d3813c4be5eb058ae27ff74167f6794b1c7a2e8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 8 Nov 2023 21:28:05 +0100 Subject: [PATCH 242/462] Update check for required ports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index a45e52a9..a3cfaedd 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -634,10 +634,8 @@ compare_port_to_service_assigned() { check_required_ports() { echo_current_diagnostic "Ports in use" - # Since Pi-hole needs 53 and 4711, check what they are being used by + # Since Pi-hole needs 53,80 and 443 check what they are being used by # so we can detect any issues - local resolver="pihole-FTL" - local web_server="pihole-FTL" local ftl="pihole-FTL" # Create an array for these ports in use ports_in_use=() @@ -656,17 +654,13 @@ check_required_ports() { local port_number port_number="$(echo "${ports_in_use[$i]}" | awk '{print $5}')" # | awk '{gsub(/^.*:/,"",$5);print $5}') - # Use a case statement to determine if the right services are using the right ports - case "$(echo "${port_number}" | rev | cut -d: -f1 | rev)" in - 53) compare_port_to_service_assigned "${resolver}" "${service_name}" "${protocol_type}:${port_number}" - ;; - 80) compare_port_to_service_assigned "${web_server}" "${service_name}" "${protocol_type}:${port_number}" - ;; - 4711) compare_port_to_service_assigned "${ftl}" "${service_name}" "${protocol_type}:${port_number}" - ;; + # Check if the right services are using the right ports + if [[ "$(echo "${port_number}" | rev | cut -d: -f1 | rev)" == @(53|80|443) ]]; then + compare_port_to_service_assigned "${ftl}" "${service_name}" "${protocol_type}:${port_number}" + else # If it's not a default port that Pi-hole needs, just print it out for the user to see - *) log_write " ${protocol_type}:${port_number} is in use by ${service_name:=}"; - esac + log_write " ${protocol_type}:${port_number} is in use by ${service_name:=}"; + fi done } From e7c89ce25fb3a8b4e59f61df63ab00838543ac59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 12 Nov 2023 20:45:45 +0100 Subject: [PATCH 243/462] No need to pass a default value to get_ftl_conf_value() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index a3cfaedd..1a3c2a34 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -80,34 +80,18 @@ PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. -# -# Args: -# 1. The key to read -# 2. The default if the setting or config does not exist get_ftl_conf_value() { local key=$1 - local default=$2 local value - # Obtain key=... setting from FTL directly - if [[ -e "$PIHOLE_FTL_CONF_FILE" ]]; then - # Constructed to return nothing when - # a) the setting is not present in the config file, or - # b) the setting is commented out (e.g. "#DBFILE=...") - value="$(pihole-FTL --config "${key}")" - fi - - # Test for missing value. Use default value in this case. - if [[ -z "$value" ]]; then - value="$default" - fi - + # Obtain setting from FTL directly + value="$(pihole-FTL --config "${key}")" echo "$value" } -PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity" "${PIHOLE_DIRECTORY}/gravity.db")" +PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity")" -PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database" "${PIHOLE_DIRECTORY}/pihole-FTL.db")" +PIHOLE_FTL_DB_FILE="$(get_ftl_conf_value "files.database")" PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole" PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE" @@ -117,8 +101,8 @@ FTL_PID="${RUN_DIRECTORY}/pihole-FTL.pid" PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log" PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*" PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log" -PIHOLE_FTL_LOG="$(get_ftl_conf_value "files.log.ftl" "${LOG_DIRECTORY}/FTL.log")" -PIHOLE_WEBSERVER_LOG="$(get_ftl_conf_value "files.log.webserver" "${LOG_DIRECTORY}/webserver.log")" +PIHOLE_FTL_LOG="$(get_ftl_conf_value "files.log.ftl")" +PIHOLE_WEBSERVER_LOG="$(get_ftl_conf_value "files.log.webserver")" RESOLVCONF="${ETC}/resolv.conf" DNSMASQ_CONF="${ETC}/dnsmasq.conf" @@ -1267,7 +1251,7 @@ analyze_pihole_log() { local pihole_log_permissions local queryLogging - queryLogging=$(pihole-FTL --config dns.queryLogging) + queryLogging="$(get_ftl_conf_value "dns.queryLogging")" if [[ "${queryLogging}" == "false" ]]; then # Inform user that logging has been disabled and pihole.log does not contain queries log_write "${INFO} Query logging is disabled" From 54ab71d8176acdfd1c0db676b8f06090cd4850ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 12 Nov 2023 21:20:03 +0100 Subject: [PATCH 244/462] Don't use hardcoded ports to check for MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 1a3c2a34..703c308b 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -618,7 +618,7 @@ compare_port_to_service_assigned() { check_required_ports() { echo_current_diagnostic "Ports in use" - # Since Pi-hole needs 53,80 and 443 check what they are being used by + # Since Pi-hole needs various ports, check what they are being used by # so we can detect any issues local ftl="pihole-FTL" # Create an array for these ports in use @@ -628,6 +628,15 @@ check_required_ports() { ports_in_use+=( "$line" ) done < <( ss --listening --numeric --tcp --udp --processes --no-header ) + local ports_configured + # Get all configured ports + ports_configured="$(pihole-FTL --config "webserver.port")" + # Remove all non-didgits, split into an array at "," + ports_configured="${ports_configured//[!0-9,]/}" + mapfile -d "," -t ports_configured < <(echo "${ports_configured}") + # Add port 53 + ports_configured+=("53") + # Now that we have the values stored, for i in "${!ports_in_use[@]}"; do # loop through them and assign some local variables @@ -639,7 +648,7 @@ check_required_ports() { port_number="$(echo "${ports_in_use[$i]}" | awk '{print $5}')" # | awk '{gsub(/^.*:/,"",$5);print $5}') # Check if the right services are using the right ports - if [[ "$(echo "${port_number}" | rev | cut -d: -f1 | rev)" == @(53|80|443) ]]; then + if [[ ${ports_configured[*]} =~ $(echo "${port_number}" | rev | cut -d: -f1 | rev) ]]; then compare_port_to_service_assigned "${ftl}" "${service_name}" "${protocol_type}:${port_number}" else # If it's not a default port that Pi-hole needs, just print it out for the user to see From b61e8be3b46bae8be8fddb8b4d7315b0afd9801c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 12 Nov 2023 21:36:18 +0100 Subject: [PATCH 245/462] Address reviewer's comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: DL6ER Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 703c308b..931e95a4 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -82,11 +82,9 @@ PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. get_ftl_conf_value() { local key=$1 - local value # Obtain setting from FTL directly - value="$(pihole-FTL --config "${key}")" - echo "$value" + pihole-FTL --config "${key}" } PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "files.gravity")" From ab09233753cd522ca84faacbe02d7bdbb7f5c40f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Nov 2023 10:17:34 +0000 Subject: [PATCH 246/462] Bump pytest-testinfra from 9.0.0 to 10.0.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 9.0.0 to 10.0.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/9.0.0...10.0.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 45c7c7c0..c3147a10 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 pytest-xdist == 3.3.1 -pytest-testinfra == 9.0.0 +pytest-testinfra == 10.0.0 tox == 4.11.3 From 3c4f217876f9ad7f8889b94d0499a480518f8a22 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Nov 2023 10:27:10 +0000 Subject: [PATCH 247/462] Bump pytest-xdist from 3.3.1 to 3.4.0 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.3.1 to 3.4.0. - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.3.1...v3.4.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 45c7c7c0..56423192 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 -pytest-xdist == 3.3.1 +pytest-xdist == 3.4.0 pytest-testinfra == 9.0.0 tox == 4.11.3 From 2e73eb36efbd129a105f5e18bfa0a6dd1b240032 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Nov 2023 10:54:16 +0000 Subject: [PATCH 248/462] Bump pytest-testinfra from 9.0.0 to 10.0.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 9.0.0 to 10.0.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/9.0.0...10.0.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 56423192..74c67fd9 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 pytest-xdist == 3.4.0 -pytest-testinfra == 9.0.0 +pytest-testinfra == 10.0.0 tox == 4.11.3 From f4b67065ccb547c374afc42274207297748deb08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Nov 2023 10:55:51 +0000 Subject: [PATCH 249/462] Bump pytest-xdist from 3.3.1 to 3.4.0 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.3.1 to 3.4.0. - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.3.1...v3.4.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index c3147a10..74c67fd9 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 -pytest-xdist == 3.3.1 +pytest-xdist == 3.4.0 pytest-testinfra == 10.0.0 tox == 4.11.3 From 19d3489bcbe5aca79ec08a38008b6d5a2f1dcaf1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 20:56:23 +0100 Subject: [PATCH 250/462] gravity_DownloadBlocklistFromUrl needs ${domain} but it was declared local before and was not passed as argument, making gravity currently depend on undefined behavior. It seems to be working well in the vast majority of cases, however, it seems we have at least one report where it is not working. Signed-off-by: DL6ER --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index edfe89a9..421ec035 100755 --- a/gravity.sh +++ b/gravity.sh @@ -464,7 +464,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" + gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" "${domain}" fi echo "" done @@ -496,7 +496,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" + local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" domain="${7}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext # Create temp file to store content on disk instead of RAM From f16cf7178186e33004bcfb44c3838a9fd2912007 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:04:46 +0100 Subject: [PATCH 251/462] ${PIHOLE_DNS_1} is gone, use the first server from pihole-FTL --config dns.upstreams instead Signed-off-by: DL6ER --- gravity.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 421ec035..90cba0f6 100755 --- a/gravity.sh +++ b/gravity.sh @@ -538,12 +538,25 @@ gravity_DownloadBlocklistFromUrl() { fi;; esac + if [[ "${blocked}" == true ]]; then - printf -v ip_addr "%s" "${PIHOLE_DNS_1%#*}" - if [[ ${PIHOLE_DNS_1} != *"#"* ]]; then + # Get first defined upstream server + local upstream + upstream="$(getFTLConfigValue dns.upstreams)" + + # Isolate first upstream server from a string like + # [ 1.2.3.4#1234, 5.6.7.8#5678, ... ] + upstream="${upstream%%,*}" + upstream="${upstream##*[}" + upstream="${upstream%%]*}" + + # Get IP address and port of this upstream server + local ip_addr port + printf -v ip_addr "%s" "${upstream%#*}" + if [[ ${upstream} != *"#"* ]]; then port=53 else - printf -v port "%s" "${PIHOLE_DNS_1#*#}" + printf -v port "%s" "${upstream#*#}" fi ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then @@ -551,7 +564,7 @@ gravity_DownloadBlocklistFromUrl() { else port=80 fi bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') - echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${PIHOLE_DNS_1} to download ${url}"; + echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; echo -ne " ${INFO} ${str} Pending..." cmd_ext="--resolve $domain:$port:$ip" fi From 4efcdf11890ca069019c5b9cb3c1206e10a7f73e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:06:09 +0100 Subject: [PATCH 252/462] Add missing double quotes to prevent globbing and word splitting Signed-off-by: DL6ER --- gravity.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 90cba0f6..7a4e52c2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -99,7 +99,7 @@ gravity_swap_databases() { # Number of available blocks on disk availableBlocks=$(stat -f --format "%a" "${gravityDIR}") # Number of blocks, used by gravity.db - gravityBlocks=$(stat --format "%b" ${gravityDBfile}) + gravityBlocks=$(stat --format "%b" "${gravityDBfile}") # Only keep the old database if available disk space is at least twice the size of the existing gravity.db. # Better be safe than sorry... oldAvail=false @@ -604,7 +604,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table file - pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "2" done="true" # Check if $listCurlBuffer is a non-zero length file @@ -614,7 +614,7 @@ gravity_DownloadBlocklistFromUrl() { # Remove curl buffer file after its use rm "${listCurlBuffer}" # Add domains to database table file - pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" # Compare lists, are they identical? compareLists "${adlistID}" "${saveLocation}" done="true" @@ -630,7 +630,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" # Add domains to database table file - pihole-FTL ${gravity_type} parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" + pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" From df7633bd1b2f4c8e653620ead7c00ab86604428a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:08:06 +0100 Subject: [PATCH 253/462] Add missing value for ${gravityDBfile_default} Signed-off-by: DL6ER --- gravity.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/gravity.sh b/gravity.sh index 7a4e52c2..64ba662e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -59,6 +59,7 @@ fi # Set this only after sourcing pihole-FTL.conf as the gravity database path may # have changed gravityDBfile="${GRAVITYDB}" +gravityDBfile_default="/etc/pihole/gravity.db" gravityTEMPfile="${GRAVITYDB}_temp" gravityDIR="$(dirname -- "${gravityDBfile}")" gravityOLDfile="${gravityDIR}/gravity_old.db" From cc333f79ccba8358b87bb38d6f3694ab3bf24809 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 22 Nov 2023 21:10:22 +0100 Subject: [PATCH 254/462] Check if this domain is blocked by Pi-hole but only if the domain is not a local file or empty Signed-off-by: DL6ER --- gravity.sh | 96 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 50 insertions(+), 46 deletions(-) diff --git a/gravity.sh b/gravity.sh index 64ba662e..4aea4d68 100755 --- a/gravity.sh +++ b/gravity.sh @@ -517,57 +517,61 @@ gravity_DownloadBlocklistFromUrl() { str="Status:" echo -ne " ${INFO} ${str} Pending..." blocked=false - case $(getFTLConfigValue dns.blocking.mode) in - "IP-NODATA-AAAA"|"IP") - # Get IP address of this domain - ip="$(dig "${domain}" +short)" - # Check if this IP matches any IP of the system - if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<< "$(ip a)") -gt 0 ]]; then - blocked=true - fi;; - "NXDOMAIN") - if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then - blocked=true - fi;; - "NODATA") - if [[ $(dig "${domain}" | grep "NOERROR" -c) -ge 1 ]] && [[ -z $(dig +short "${domain}") ]]; then - blocked=true - fi;; - "NULL"|*) - if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then - blocked=true - fi;; - esac + # Check if this domain is blocked by Pi-hole but only if the domain is not a + # local file or empty + if [[ $url != "file"* ]] && [[ -n "${domain}" ]]; then + case $(getFTLConfigValue dns.blocking.mode) in + "IP-NODATA-AAAA"|"IP") + # Get IP address of this domain + ip="$(dig "${domain}" +short)" + # Check if this IP matches any IP of the system + if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<< "$(ip a)") -gt 0 ]]; then + blocked=true + fi;; + "NXDOMAIN") + if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then + blocked=true + fi;; + "NODATA") + if [[ $(dig "${domain}" | grep "NOERROR" -c) -ge 1 ]] && [[ -z $(dig +short "${domain}") ]]; then + blocked=true + fi;; + "NULL"|*) + if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then + blocked=true + fi;; + esac - if [[ "${blocked}" == true ]]; then - # Get first defined upstream server - local upstream - upstream="$(getFTLConfigValue dns.upstreams)" + if [[ "${blocked}" == true ]]; then + # Get first defined upstream server + local upstream + upstream="$(getFTLConfigValue dns.upstreams)" - # Isolate first upstream server from a string like - # [ 1.2.3.4#1234, 5.6.7.8#5678, ... ] - upstream="${upstream%%,*}" - upstream="${upstream##*[}" - upstream="${upstream%%]*}" + # Isolate first upstream server from a string like + # [ 1.2.3.4#1234, 5.6.7.8#5678, ... ] + upstream="${upstream%%,*}" + upstream="${upstream##*[}" + upstream="${upstream%%]*}" - # Get IP address and port of this upstream server - local ip_addr port - printf -v ip_addr "%s" "${upstream%#*}" - if [[ ${upstream} != *"#"* ]]; then - port=53 - else - printf -v port "%s" "${upstream#*#}" + # Get IP address and port of this upstream server + local ip_addr port + printf -v ip_addr "%s" "${upstream%#*}" + if [[ ${upstream} != *"#"* ]]; then + port=53 + else + printf -v port "%s" "${upstream#*#}" + fi + ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) + if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then + port=443; + else port=80 + fi + bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') + echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; + echo -ne " ${INFO} ${str} Pending..." + cmd_ext="--resolve $domain:$port:$ip" fi - ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) - if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then - port=443; - else port=80 - fi - bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') - echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; - echo -ne " ${INFO} ${str} Pending..." - cmd_ext="--resolve $domain:$port:$ip" fi # shellcheck disable=SC2086 From c785667efea9e579db37c7d450bd3fbe9e7612a0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 23 Nov 2023 09:56:43 +0100 Subject: [PATCH 255/462] Trim leading and trailing spaces and tabs in upstream servers (if any) Signed-off-by: DL6ER --- gravity.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 4aea4d68..20ad6215 100755 --- a/gravity.sh +++ b/gravity.sh @@ -553,6 +553,9 @@ gravity_DownloadBlocklistFromUrl() { upstream="${upstream%%,*}" upstream="${upstream##*[}" upstream="${upstream%%]*}" + # Trim leading and trailing spaces and tabs + upstream="${upstream#"${upstream%%[![:space:]]*}"}" + upstream="${upstream%"${upstream##*[![:space:]]}"}" # Get IP address and port of this upstream server local ip_addr port @@ -567,8 +570,7 @@ gravity_DownloadBlocklistFromUrl() { port=443; else port=80 fi - bad_list=$(pihole -q -adlist "${domain}" | head -n1 | awk -F 'Match found in ' '{print $2}') - echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by ${bad_list%:}. Using DNS on ${upstream} to download ${url}"; + echo -e "${OVER} ${CROSS} ${str} ${domain} is blocked by one of your lists. Using DNS server ${upstream} instead"; echo -ne " ${INFO} ${str} Pending..." cmd_ext="--resolve $domain:$port:$ip" fi From a6565bf9a17ad4998b00ca239be9044be7f51674 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 23 Nov 2023 22:07:31 +0100 Subject: [PATCH 256/462] Support special webserver.port ports ending in "s" (secure) and "r" (redirect) Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 43 ++++++++++++++++++++++++++++------------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 449f146f..000c0717 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -21,14 +21,31 @@ TestAPIAvailability() { # as we are running locally, we can get the port value from FTL directly - PORT="$(pihole-FTL --config webserver.port)" - PORT="${PORT%%,*}" + local ports port availabilityResonse + ports="$(pihole-FTL --config webserver.port)" + port="${ports%%,*}" - availabilityResonse=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${PORT}/api/auth") + # if the port ends with an "s", it is a secure connection + if [ "${port#"${port%?}"}" = "s" ]; then + # remove the "s" from the port + API_PROT="https" + API_PORT="${port%?}" + elif [ "${port#"${port%?}"}" = "r" ]; then + # if the port ends in "r", it is a redirect + API_PROT="http" + # remove the "r" from the port + API_PORT="${port%?}" + else + API_PROT="http" + API_PORT="${port}" + fi - # test if http status code was 200 (OK) or 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then - echo "API not available at: http://localhost:${PORT}/api" + API_URL="${API_PROT}://localhost:${API_PORT}/api" + availabilityResonse=$(curl -skSL -o /dev/null -w "%{http_code}" "${API_URL}/auth") + + # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + echo "API not available at: ${API_URL}" echo "Exiting." exit 1 fi @@ -54,15 +71,15 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl --silent -X POST "http://localhost:${PORT}/api/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skSL -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" exit 1 fi - # obtain validity and session ID from session response - validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) - SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) + # obtain validity and session ID from session response + validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) + SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) } DeleteSession() { @@ -70,7 +87,7 @@ DeleteSession() { # SID is not null (successful authenthication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -s -o /dev/null -w "%{http_code}" -X DELETE "http://localhost:${PORT}/api/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skSL -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; @@ -84,14 +101,14 @@ DeleteSession() { GetFTLData() { local data response status # get the data from querying the API as well as the http status code - response=$(curl -s -w "%{http_code}" -X GET "http://localhost:${PORT}/api$1" -H "Accept: application/json" -H "sid: ${SID}" ) + response=$(curl -skSL -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) # status are the last 3 characters status=$(printf %s "${response#"${response%???}"}") # data is everything from response without the last 3 characters data=$(printf %s "${response%???}") - if [ "${status}" = 200 ]; then + if [ "${status}" = 200 ] || [ "${status}" = 308 ]; then # response OK echo "${data}" elif [ "${status}" = 000 ]; then From 1276242a4ea3164cf2c24c9727fa7396b1495936 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Nov 2023 10:16:35 +0000 Subject: [PATCH 257/462] Bump pytest-xdist from 3.4.0 to 3.5.0 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 74c67fd9..799e3fad 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 -pytest-xdist == 3.4.0 +pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.11.3 From 9d57f6493751d319440727f927f099887decb886 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 Nov 2023 10:58:21 +0000 Subject: [PATCH 258/462] Bump pytest-xdist from 3.4.0 to 3.5.0 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.4.0...v3.5.0) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 74c67fd9..799e3fad 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 7.4.3 -pytest-xdist == 3.4.0 +pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.11.3 From 974fea592df99d97332763b6f9cf9812a6c907a4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 00:05:55 +0100 Subject: [PATCH 259/462] Iterate over ports, skip redirected ports Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 64 +++++++++++++++++++++++++++-------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 000c0717..b50e416a 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -25,26 +25,48 @@ TestAPIAvailability() { ports="$(pihole-FTL --config webserver.port)" port="${ports%%,*}" - # if the port ends with an "s", it is a secure connection - if [ "${port#"${port%?}"}" = "s" ]; then - # remove the "s" from the port - API_PROT="https" - API_PORT="${port%?}" - elif [ "${port#"${port%?}"}" = "r" ]; then - # if the port ends in "r", it is a redirect - API_PROT="http" - # remove the "r" from the port - API_PORT="${port%?}" - else - API_PROT="http" - API_PORT="${port}" - fi + # Iterate over comma separated list of ports + while [ "${port}" != "${ports}" ]; do + # if the port ends with an "s", it is a secure connection + if [ "${port#"${port%?}"}" = "s" ]; then + # remove the "s" from the port + API_PROT="https" + API_PORT="${port%?}" + elif [ "${port#"${port%?}"}" = "r" ]; then + # Ignore this port + API_PORT="0" + else + API_PROT="http" + API_PORT="${port}" + fi - API_URL="${API_PROT}://localhost:${API_PORT}/api" - availabilityResonse=$(curl -skSL -o /dev/null -w "%{http_code}" "${API_URL}/auth") + if [ ! "${API_PORT}" = "0" ]; then + # If the port is of form "ip:port", we need to remove everything before + # the last ":" in the string, e.g., "[::]:80" -> "80" + if [ "${API_PORT#*:}" != "${API_PORT}" ]; then + API_PORT="${API_PORT##*:}" + fi - # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + API_URL="${API_PROT}://localhost:${API_PORT}/api" + availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") + + # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + API_PORT="0" + else + # API is available at this port/protocol combination + break + fi + fi + + # remove the first port from the list + ports="${ports#*,}" + # get the next port + port="${ports%%,*}" + done + + # if API_PORT is 0, no working API port was found + if [ "${API_PORT}" = "0" ]; then echo "API not available at: ${API_URL}" echo "Exiting." exit 1 @@ -71,7 +93,7 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl -skSL -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skS -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" @@ -87,7 +109,7 @@ DeleteSession() { # SID is not null (successful authenthication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -skSL -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; @@ -101,7 +123,7 @@ DeleteSession() { GetFTLData() { local data response status # get the data from querying the API as well as the http status code - response=$(curl -skSL -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) + response=$(curl -skS -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) # status are the last 3 characters status=$(printf %s "${response#"${response%???}"}") From 6016131280ce5f24cc53fa7989026f593f4d2d4f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 28 Nov 2023 22:59:49 +0100 Subject: [PATCH 260/462] Ensure we also check the last port Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index b50e416a..2952fb43 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -26,16 +26,18 @@ TestAPIAvailability() { port="${ports%%,*}" # Iterate over comma separated list of ports - while [ "${port}" != "${ports}" ]; do + while [ -n "${ports}" ]; do # if the port ends with an "s", it is a secure connection if [ "${port#"${port%?}"}" = "s" ]; then # remove the "s" from the port API_PROT="https" API_PORT="${port%?}" elif [ "${port#"${port%?}"}" = "r" ]; then - # Ignore this port + # Ignore this port, the client may not be able to follow the + # redirected target when FTL is not used as local resolver API_PORT="0" else + # otherwise it is an insecure (plain HTTP) connection API_PROT="http" API_PORT="${port}" fi @@ -50,8 +52,9 @@ TestAPIAvailability() { API_URL="${API_PROT}://localhost:${API_PORT}/api" availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") - # test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # API is not available at this port/protocol combination API_PORT="0" else # API is available at this port/protocol combination @@ -59,9 +62,9 @@ TestAPIAvailability() { fi fi - # remove the first port from the list + # If the loop has not been broken, remove the first port from the list + # and get the next port ports="${ports#*,}" - # get the next port port="${ports%%,*}" done @@ -192,3 +195,6 @@ secretRead() { # restore original terminal settings stty "${stty_orig}" } + + +TestAPIAvailability From 96bf07863f6bdd4d9cad91d5444fef5f85d47b61 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 09:10:06 +0100 Subject: [PATCH 261/462] Use CHAOS TXT local.api.txt instead of trying to parse pihole-FTL --config webserver.ports Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 90 +++++++++++++++++++-------------------- advanced/Scripts/query.sh | 6 +-- 2 files changed, 47 insertions(+), 49 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 2952fb43..46da37cd 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -21,62 +21,60 @@ TestAPIAvailability() { # as we are running locally, we can get the port value from FTL directly - local ports port availabilityResonse - ports="$(pihole-FTL --config webserver.port)" - port="${ports%%,*}" + local chaos_api_list availabilityResonse - # Iterate over comma separated list of ports - while [ -n "${ports}" ]; do - # if the port ends with an "s", it is a secure connection - if [ "${port#"${port%?}"}" = "s" ]; then - # remove the "s" from the port - API_PROT="https" - API_PORT="${port%?}" - elif [ "${port#"${port%?}"}" = "r" ]; then - # Ignore this port, the client may not be able to follow the - # redirected target when FTL is not used as local resolver - API_PORT="0" + # Query the API URLs from FTL using CHAOS TXT local.api.ftl + # The result is a space-separated enumeration of full URLs + # e.g., "http://localhost:80/api" "https://localhost:443/api" + chaos_api_list="$(dig +short chaos txt local.api.ftl @127.0.0.1)" + + # If the query was not successful, the variable is empty + if [ -z "${chaos_api_list}" ]; then + echo "API not available. Please check connectivity" + exit 1 + fi + + # Iterate over space-separated list of URLs + while [ -n "${chaos_api_list}" ]; do + # Get the first URL + API_URL="${chaos_api_list%% *}" + # Strip leading and trailing quotes + API_URL="${API_URL%\"}" + API_URL="${API_URL#\"}" + + # Test if the API is available at this URL + availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") + + # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # API is not available at this port/protocol combination + API_PORT="" else - # otherwise it is an insecure (plain HTTP) connection - API_PROT="http" - API_PORT="${port}" + # API is available at this URL combination + break fi - if [ ! "${API_PORT}" = "0" ]; then - # If the port is of form "ip:port", we need to remove everything before - # the last ":" in the string, e.g., "[::]:80" -> "80" - if [ "${API_PORT#*:}" != "${API_PORT}" ]; then - API_PORT="${API_PORT##*:}" - fi + # Remove the first URL from the list + local last_api_list + last_api_list="${chaos_api_list}" + chaos_api_list="${chaos_api_list#* }" - API_URL="${API_PROT}://localhost:${API_PORT}/api" - availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}/auth") - - # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then - # API is not available at this port/protocol combination - API_PORT="0" - else - # API is available at this port/protocol combination - break - fi + # If the list did not change, we are at the last element + if [ "${last_api_list}" = "${chaos_api_list}" ]; then + # Remove the last element + chaos_api_list="" fi - - # If the loop has not been broken, remove the first port from the list - # and get the next port - ports="${ports#*,}" - port="${ports%%,*}" done - # if API_PORT is 0, no working API port was found - if [ "${API_PORT}" = "0" ]; then + # if API_PORT is empty, no working API port was found + if [ -n "${API_PORT}" ]; then echo "API not available at: ${API_URL}" echo "Exiting." exit 1 fi } -Authenthication() { +Authentication() { # Try to authenticate LoginAPI @@ -96,7 +94,7 @@ Authenthication() { } LoginAPI() { - sessionResponse="$(curl -skS -X POST "${API_URL}/auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" + sessionResponse="$(curl -skS -X POST "${API_URL}auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then echo "No response from FTL server. Please check connectivity" @@ -108,11 +106,11 @@ LoginAPI() { } DeleteSession() { - # if a valid Session exists (no password required or successful authenthication) and - # SID is not null (successful authenthication only), delete the session + # if a valid Session exists (no password required or successful Authentication) and + # SID is not null (successful Authentication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then # Try to delete the session. Omit the output, but get the http status code - deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}/auth" -H "Accept: application/json" -H "sid: ${SID}") + deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 2279df85..62d29d5b 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -121,14 +121,14 @@ Main(){ # or b) for the /search endpoint (webserver.api.searchAPIauth) no authentication is required. # Therefore, we try to query directly without authentication but do authenticat if 401 is returned - data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") if [ "${data}" = 401 ]; then # Unauthenticated, so authenticate with the FTL server required - Authenthication + Authentication # send query again - data=$(GetFTLData "/search/${domain}?N=${max_results}&partial=${partial}") + data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") fi GenerateOutput "${data}" From 29d010dc2ccc04cd6563d8154415e2070b5bd9a7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 1 Dec 2023 10:21:02 +0100 Subject: [PATCH 262/462] Use files.gravity_tmp as temporary directory for the intermediate lists Signed-off-by: DL6ER --- gravity.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index edfe89a9..8dc8edd8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -41,6 +41,7 @@ VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" GRAVITYDB=$(getFTLConfigValue files.gravity) +GRAVITY_TMPDIR=$(getFTLConfigValue files.gravity_tmp) gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" @@ -48,9 +49,7 @@ domainsExtension="domains" curl_connect_timeout=10 -# Set up tmp dir variable in case it's not configured -: "${GRAVITY_TMPDIR:=/tmp}" - +# Check gravity temp directory if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then echo -e " ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}" GRAVITY_TMPDIR="/tmp" From bfc824f2ff05f56d24697230463a3ec0bf2a25cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Dec 2023 10:04:50 +0000 Subject: [PATCH 263/462] Bump tox from 4.11.3 to 4.11.4 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.3 to 4.11.4. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.3...4.11.4) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 799e3fad..bfc6d027 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.3 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.11.3 +tox == 4.11.4 From f5fe550a2e27e5b81ffbe55d3f209c890dda3b94 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Dec 2023 10:35:21 +0000 Subject: [PATCH 264/462] Bump tox from 4.11.3 to 4.11.4 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.3 to 4.11.4. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.3...4.11.4) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 799e3fad..bfc6d027 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.3 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.11.3 +tox == 4.11.4 From 32a741b5c74bab31ed5c02f5ba33f65f631fea7f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 2 Dec 2023 22:42:36 +0100 Subject: [PATCH 265/462] We do not follow 308 but FTL also doesn't suggest it Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 46da37cd..b7bc2a86 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -25,7 +25,7 @@ TestAPIAvailability() { # Query the API URLs from FTL using CHAOS TXT local.api.ftl # The result is a space-separated enumeration of full URLs - # e.g., "http://localhost:80/api" "https://localhost:443/api" + # e.g., "http://localhost:80/api/" "https://localhost:443/api/" chaos_api_list="$(dig +short chaos txt local.api.ftl @127.0.0.1)" # If the query was not successful, the variable is empty @@ -45,8 +45,8 @@ TestAPIAvailability() { # Test if the API is available at this URL availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") - # Test if http status code was 200 (OK), 308 (redirect, we follow) 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 308 ] && [ ! "${availabilityResonse}" = 401 ]; then + # Test if http status code was 200 (OK) or 401 (authentication required) + if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then # API is not available at this port/protocol combination API_PORT="" else @@ -131,7 +131,7 @@ GetFTLData() { # data is everything from response without the last 3 characters data=$(printf %s "${response%???}") - if [ "${status}" = 200 ] || [ "${status}" = 308 ]; then + if [ "${status}" = 200 ]; then # response OK echo "${data}" elif [ "${status}" = 000 ]; then @@ -193,6 +193,3 @@ secretRead() { # restore original terminal settings stty "${stty_orig}" } - - -TestAPIAvailability From 2681835f94ec072e183d6cd6de267f25b93fb0a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 6 Dec 2023 00:21:04 +0100 Subject: [PATCH 266/462] Treat FTL return data as strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 2 +- advanced/Scripts/query.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 449f146f..bf6e3654 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -93,7 +93,7 @@ GetFTLData() { if [ "${status}" = 200 ]; then # response OK - echo "${data}" + printf %s "${data}" elif [ "${status}" = 000 ]; then # connection lost echo "000" diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 2279df85..789efe75 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -46,14 +46,14 @@ GenerateOutput(){ data="${1}" # construct a new json for the list results where each object contains the domain and the related type - lists_data=$(echo "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') + lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') # construct a new json for the gravity results where each object contains the adlist URL and the related domains - gravity_data=$(echo "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') + gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') # number of objects in each json - num_gravity=$(echo "${gravity_data}" | jq length ) - num_lists=$(echo "${lists_data}" | jq length ) + num_gravity=$(printf %s "${gravity_data}" | jq length ) + num_lists=$(printf %s "${lists_data}" | jq length ) if [ "${partial}" = true ]; then search_type_str="partially" @@ -66,7 +66,7 @@ GenerateOutput(){ if [ "${num_lists}" -gt 0 ]; then # Convert the data to a csv, each line is a "domain,type" string # not using jq's @csv here as it quotes each value individually - lists_data_csv=$(echo "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")' ) + lists_data_csv=$(printf %s "${lists_data}" | jq --raw-output '.[] | [.domain, .type] | join(",")' ) # Generate output for each csv line, separating line in a domain and type substring at the ',' echo "${lists_data_csv}" | while read -r line; do @@ -79,7 +79,7 @@ GenerateOutput(){ if [ "${num_gravity}" -gt 0 ]; then # Convert the data to a csv, each line is a "URL,domain,domain,...." string # not using jq's @csv here as it quotes each value individually - gravity_data_csv=$(echo "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")' ) + gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")' ) # Generate line-by-line output for each csv line echo "${gravity_data_csv}" | while read -r line; do From 159817b7e286c4015030bfe61c6100b7d5b2c028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 6 Dec 2023 20:35:26 +0100 Subject: [PATCH 267/462] Use development-v6 as ftl branch for binary test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 7d9d15f6..62588c0a 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -465,7 +465,7 @@ def test_FTL_development_binary_installed_and_responsive_no_errors(host): source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - echo "development" > /etc/pihole/ftlbranch + echo "development-v6" > /etc/pihole/ftlbranch binary="pihole-FTL${funcOutput##*pihole-FTL}" theRest="${funcOutput%pihole-FTL*}" FTLdetect "${binary}" "${theRest}" From 16180e4b234510d3c52545f817164fd9873448a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 10:36:44 +0000 Subject: [PATCH 268/462] Bump actions/setup-python from 4.7.1 to 5.0.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.1 to 5.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.1...v5.0.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b070c982..35ed0c30 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -68,7 +68,7 @@ jobs: uses: actions/checkout@v4.1.1 - name: Set up Python 3.10 - uses: actions/setup-python@v4.7.1 + uses: actions/setup-python@v5.0.0 with: python-version: "3.10" From c34464d1e8f6e0ba5731171791d9a31d801dfa5c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 10:36:49 +0000 Subject: [PATCH 269/462] Bump actions/stale from 8.0.0 to 9.0.0 Bumps [actions/stale](https://github.com/actions/stale) from 8.0.0 to 9.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v8.0.0...v9.0.0) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- .github/workflows/stale_pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 0e149c79..095d7358 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -17,7 +17,7 @@ jobs: issues: write steps: - - uses: actions/stale@v8.0.0 + - uses: actions/stale@v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 diff --git a/.github/workflows/stale_pr.yml b/.github/workflows/stale_pr.yml index 2db2a25d..96650818 100644 --- a/.github/workflows/stale_pr.yml +++ b/.github/workflows/stale_pr.yml @@ -17,7 +17,7 @@ jobs: pull-requests: write steps: - - uses: actions/stale@v8.0.0 + - uses: actions/stale@v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Do not automatically mark PR/issue as stale From 8a71e4253e0733fec4a4cb805ea219ff8ec8dab2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 17:39:18 +0000 Subject: [PATCH 270/462] Bump actions/stale from 8.0.0 to 9.0.0 Bumps [actions/stale](https://github.com/actions/stale) from 8.0.0 to 9.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/v8.0.0...v9.0.0) --- updated-dependencies: - dependency-name: actions/stale dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yml | 2 +- .github/workflows/stale_pr.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d9de09d2..74ee9bef 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -17,7 +17,7 @@ jobs: issues: write steps: - - uses: actions/stale@v8.0.0 + - uses: actions/stale@v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 30 diff --git a/.github/workflows/stale_pr.yml b/.github/workflows/stale_pr.yml index 2db2a25d..96650818 100644 --- a/.github/workflows/stale_pr.yml +++ b/.github/workflows/stale_pr.yml @@ -17,7 +17,7 @@ jobs: pull-requests: write steps: - - uses: actions/stale@v8.0.0 + - uses: actions/stale@v9.0.0 with: repo-token: ${{ secrets.GITHUB_TOKEN }} # Do not automatically mark PR/issue as stale From e49d7fa5f1dd19a80c798318a270e80f0a6d9415 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Dec 2023 17:39:42 +0000 Subject: [PATCH 271/462] Bump actions/setup-python from 4.7.1 to 5.0.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4.7.1 to 5.0.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4.7.1...v5.0.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8166d253..ec4da6bb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -75,7 +75,7 @@ jobs: uses: actions/checkout@v4.1.1 - name: Set up Python 3.10 - uses: actions/setup-python@v4.7.1 + uses: actions/setup-python@v5.0.0 with: python-version: "3.10" From a87d1bbc4fcf107753324f551ac54bfe7bc45a58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 14:22:43 +0100 Subject: [PATCH 272/462] Remove pihole.sudo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole.sudo | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 advanced/Templates/pihole.sudo diff --git a/advanced/Templates/pihole.sudo b/advanced/Templates/pihole.sudo deleted file mode 100644 index 708309be..00000000 --- a/advanced/Templates/pihole.sudo +++ /dev/null @@ -1,9 +0,0 @@ -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Allows the WebUI to use Pi-hole commands -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. -# From 54186a63eee39a2e420dda8d170807e3492bbc3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 14:33:13 +0100 Subject: [PATCH 273/462] Remove test for sudo file as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_any_automated_install.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 62588c0a..c53070dc 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -174,10 +174,6 @@ def test_installPihole_fresh_install_readableFiles(host): ) actual_rc = host.run(check_man).rc assert exit_status_success == actual_rc - # check not readable sudoers file - check_sudo = test_cmd.format("r", "/etc/sudoers.d/pihole", piholeuser) - actual_rc = host.run(check_sudo).rc - assert exit_status_success != actual_rc # check not readable cron file check_sudo = test_cmd.format("x", "/etc/cron.d/", piholeuser) actual_rc = host.run(check_sudo).rc From 274d4c263c2ec4c8ef3d3b4a55bae45aa3fb694b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 21:23:39 +0100 Subject: [PATCH 274/462] Add bash-completion to PIHOLE_DEPS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 72f00992..f5bf15e6 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -332,7 +332,7 @@ package_manager_detect() { # Packages required to run this install script INSTALLER_DEPS=(git iproute2 dialog ca-certificates) # Packages required to run Pi-hole - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq lshw) + PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq lshw bash-completion) # If apt-get is not found, check for rpm. elif is_command rpm ; then @@ -349,7 +349,7 @@ package_manager_detect() { PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) - PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq lshw) + PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq lshw bash-completion) # If neither apt-get or yum/dnf package managers were found else From fe7299323c389ca86fe941ecadaee6ce082695a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 21:40:03 +0100 Subject: [PATCH 275/462] Update bash_completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/bash-completion/pihole | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 305a3f5b..b2740724 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -1,5 +1,5 @@ _pihole() { - local cur prev opts opts_admin opts_checkout opts_debug opts_interface opts_logging opts_privacy opts_query opts_update opts_version + local cur prev opts opts_checkout opts_debug opts_logging opts_query opts_update opts_version COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" @@ -7,17 +7,13 @@ _pihole() { case "${prev}" in "pihole") - opts="admin blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" + opts="blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; "whitelist"|"blacklist"|"wildcard"|"regex") opts_lists="\--delmode \--noreload \--quiet \--list \--nuke" COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) ;; - "admin") - opts_admin="celsius fahrenheit interface kelvin password privacylevel" - COMPREPLY=( $(compgen -W "${opts_admin}" -- ${cur}) ) - ;; "checkout") opts_checkout="core ftl web master dev" COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) @@ -31,7 +27,7 @@ _pihole() { COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) ) ;; "query") - opts_query="-adlist -all -exact" + opts_query="--partial --all" COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) ) ;; "updatePihole"|"-up") @@ -41,23 +37,7 @@ _pihole() { "version") opts_version="\--admin \--current \--ftl \--hash \--latest \--pihole" COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) - ;; - "interface") - if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then - opts_interface="$(cat /proc/net/dev | cut -d: -s -f1)" - COMPREPLY=( $(compgen -W "${opts_interface}" -- ${cur}) ) - else - return 1 - fi - ;; - "privacylevel") - if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then - opts_privacy="0 1 2 3" - COMPREPLY=( $(compgen -W "${opts_privacy}" -- ${cur}) ) - else - return 1 - fi - ;; + ;; "core"|"admin"|"ftl") if [[ "$prev2" == "checkout" ]]; then opts_checkout="master dev" From 6d02d4056f6abd63ff3d9479979502b6523f715e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 22:01:30 +0100 Subject: [PATCH 276/462] Rename option --admin to --web in version function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/version.sh | 16 ++++++++-------- advanced/bash-completion/pihole | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index e3b4a6ae..2983c04e 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -24,7 +24,7 @@ fi getLocalVersion() { case ${1} in "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "web" ) echo "${WEB_VERSION:=N/A}";; + "web" ) echo "${WEB_VERSION:=N/A}";; "FTL" ) echo "${FTL_VERSION:=N/A}";; esac } @@ -32,7 +32,7 @@ getLocalVersion() { getLocalHash() { case ${1} in "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "web" ) echo "${WEB_HASH:=N/A}";; + "web" ) echo "${WEB_HASH:=N/A}";; "FTL" ) echo "${FTL_HASH:=N/A}";; esac } @@ -40,7 +40,7 @@ getLocalHash() { getRemoteHash(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; + "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; esac } @@ -48,7 +48,7 @@ getRemoteHash(){ getRemoteVersion(){ case ${1} in "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; + "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; esac } @@ -56,7 +56,7 @@ getRemoteVersion(){ getLocalBranch(){ case ${1} in "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "web" ) echo "${WEB_BRANCH:=N/A}";; + "web" ) echo "${WEB_BRANCH:=N/A}";; "FTL" ) echo "${FTL_BRANCH:=N/A}";; esac } @@ -114,11 +114,11 @@ defaultOutput() { helpFunc() { echo "Usage: pihole -v [repo | option] [option] Example: 'pihole -v -p -l' -Show Pi-hole, Admin Console & FTL versions +Show Pi-hole, Web Console & FTL versions Repositories: -p, --pihole Only retrieve info regarding Pi-hole repository - -a, --admin Only retrieve info regarding web repository + -w, --web Only retrieve info regarding web repository -f, --ftl Only retrieve info regarding FTL repository Options: @@ -131,7 +131,7 @@ Options: case "${1}" in "-p" | "--pihole" ) shift; versionOutput "Pi-hole" "$@";; - "-a" | "--admin" ) shift; versionOutput "web" "$@";; + "-w" | "--web" ) shift; versionOutput "web" "$@";; "-f" | "--ftl" ) shift; versionOutput "FTL" "$@";; "-h" | "--help" ) helpFunc;; * ) defaultOutput "$@";; diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index b2740724..4fe8f83a 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -35,7 +35,7 @@ _pihole() { COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) ;; "version") - opts_version="\--admin \--current \--ftl \--hash \--latest \--pihole" + opts_version="\--web \--current \--ftl \--hash \--latest \--pihole" COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) ;; "core"|"admin"|"ftl") From c90a27c5096d53f4be21044fdd71cb380806bc3f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 9 Dec 2023 23:06:50 +0100 Subject: [PATCH 277/462] Add "-ni" to all sqlite3 invocations Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 34 ++++++++-------- advanced/Scripts/list.sh | 20 +++++----- advanced/Scripts/piholeARPTable.sh | 4 +- advanced/Scripts/piholeDebug.sh | 12 +++--- advanced/Scripts/piholeLogFlush.sh | 2 +- gravity.sh | 39 +++++++++---------- 6 files changed, 55 insertions(+), 56 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index e36d9b1e..7c4deaa7 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -19,13 +19,13 @@ upgrade_gravityDB(){ auditFile="${piholeDir}/auditlog.list" # Get database version - version="$(pihole-FTL sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" + version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" if [[ "$version" == "1" ]]; then # This migration script upgrades the gravity.db file by # adding the domain_audit table echo -e " ${INFO} Upgrading gravity database from version 1 to 2" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/1_to_2.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql" version=2 # Store audit domains in database table @@ -40,28 +40,28 @@ upgrade_gravityDB(){ # renaming the regex table to regex_blacklist, and # creating a new regex_whitelist table + corresponding linking table and views echo -e " ${INFO} Upgrading gravity database from version 2 to 3" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/2_to_3.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql" version=3 fi if [[ "$version" == "3" ]]; then # This migration script unifies the formally separated domain # lists into a single table with a UNIQUE domain constraint echo -e " ${INFO} Upgrading gravity database from version 3 to 4" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/3_to_4.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql" version=4 fi if [[ "$version" == "4" ]]; then # This migration script upgrades the gravity and list views # implementing necessary changes for per-client blocking echo -e " ${INFO} Upgrading gravity database from version 4 to 5" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/4_to_5.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql" version=5 fi if [[ "$version" == "5" ]]; then # This migration script upgrades the adlist view # to return an ID used in gravity.sh echo -e " ${INFO} Upgrading gravity database from version 5 to 6" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/5_to_6.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql" version=6 fi if [[ "$version" == "6" ]]; then @@ -69,7 +69,7 @@ upgrade_gravityDB(){ # which is automatically associated to all clients not # having their own group assignments echo -e " ${INFO} Upgrading gravity database from version 6 to 7" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/6_to_7.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql" version=7 fi if [[ "$version" == "7" ]]; then @@ -77,21 +77,21 @@ upgrade_gravityDB(){ # to ensure uniqueness on the group name # We also add date_added and date_modified columns echo -e " ${INFO} Upgrading gravity database from version 7 to 8" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/7_to_8.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql" version=8 fi if [[ "$version" == "8" ]]; then # This migration fixes some issues that were introduced # in the previous migration script. echo -e " ${INFO} Upgrading gravity database from version 8 to 9" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/8_to_9.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql" version=9 fi if [[ "$version" == "9" ]]; then # This migration drops unused tables and creates triggers to remove # obsolete groups assignments when the linked items are deleted echo -e " ${INFO} Upgrading gravity database from version 9 to 10" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/9_to_10.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql" version=10 fi if [[ "$version" == "10" ]]; then @@ -101,44 +101,44 @@ upgrade_gravityDB(){ # to keep the copying process generic (needs the same columns in both the # source and the destination databases). echo -e " ${INFO} Upgrading gravity database from version 10 to 11" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/10_to_11.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql" version=11 fi if [[ "$version" == "11" ]]; then # Rename group 0 from "Unassociated" to "Default" echo -e " ${INFO} Upgrading gravity database from version 11 to 12" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/11_to_12.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql" version=12 fi if [[ "$version" == "12" ]]; then # Add column date_updated to adlist table echo -e " ${INFO} Upgrading gravity database from version 12 to 13" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/12_to_13.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql" version=13 fi if [[ "$version" == "13" ]]; then # Add columns number and status to adlist table echo -e " ${INFO} Upgrading gravity database from version 13 to 14" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/13_to_14.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql" version=14 fi if [[ "$version" == "14" ]]; then # Changes the vw_adlist created in 5_to_6 echo -e " ${INFO} Upgrading gravity database from version 14 to 15" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql" version=15 fi if [[ "$version" == "15" ]]; then # Add column abp_entries to adlist table echo -e " ${INFO} Upgrading gravity database from version 15 to 16" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/15_to_16.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/15_to_16.sql" version=16 fi if [[ "$version" == "16" ]]; then # Add antigravity table # Add column type to adlist table (to support adlist types) echo -e " ${INFO} Upgrading gravity database from version 16 to 17" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/16_to_17.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql" version=17 fi } diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index b76a7ef7..76558e58 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -150,18 +150,18 @@ AddDomain() { domain="$1" # Is the domain in the list we want to add it to? - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" requestedListname="$(GetListnameFromTypeId "${typeId}")" if [[ "${num}" -ne 0 ]]; then - existingTypeId="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" + existingTypeId="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" if [[ "${existingTypeId}" == "${typeId}" ]]; then if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${requestedListname}, no need to add!" fi else existingListname="$(GetListnameFromTypeId "${existingTypeId}")" - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!" fi @@ -177,10 +177,10 @@ AddDomain() { # Insert only the domain here. The enabled and date_added fields will be filled # with their default values (enabled = true, date_added = current timestamp) if [[ -z "${comment}" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" else # also add comment when variable has been set through the "--comment" option - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" fi } @@ -189,7 +189,7 @@ RemoveDomain() { domain="$1" # Is the domain in the list we want to remove it from? - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" requestedListname="$(GetListnameFromTypeId "${typeId}")" @@ -206,14 +206,14 @@ RemoveDomain() { fi reload=true # Remove it from the current list - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" } Displaylist() { local count num_pipes domain enabled status nicedate requestedListname requestedListname="$(GetListnameFromTypeId "${typeId}")" - data="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" + data="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" if [[ -z $data ]]; then echo -e "Not showing empty list" @@ -251,10 +251,10 @@ Displaylist() { } NukeList() { - count=$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") + count=$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") listname="$(GetListnameFromTypeId "${typeId}")" if [ "$count" -gt 0 ];then - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" echo " ${TICK} Removed ${count} domain(s) from the ${listname}" else echo " ${INFO} ${listname} already empty. Nothing to do!" diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index 5daa025d..b92dd124 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -39,7 +39,7 @@ flushARP(){ # Truncate network_addresses table in pihole-FTL.db # This needs to be done before we can truncate the network table due to # foreign key constraints - if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then + if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network_addresses table" echo " Database location: ${DBFILE}" echo " Output: ${output}" @@ -47,7 +47,7 @@ flushARP(){ fi # Truncate network table in pihole-FTL.db - if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then + if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network table" echo " Database location: ${DBFILE}" echo " Output: ${output}" diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 931e95a4..aa0e61c4 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -718,7 +718,7 @@ dig_at() { # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") + random_url=$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") # Fallback if no non-ABP style domains were found if [ -z "${random_url}" ]; then random_url="flurry.com" @@ -1064,7 +1064,7 @@ show_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \ + pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1089,7 +1089,7 @@ show_FTL_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - pihole-FTL sqlite3 "${PIHOLE_FTL_DB_FILE}" \ + pihole-FTL sqlite3 -ni "${PIHOLE_FTL_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1155,7 +1155,7 @@ analyze_gravity_list() { fi show_db_entries "Info table" "SELECT property,value FROM info" "20 40" - gravity_updated_raw="$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" + gravity_updated_raw="$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" gravity_updated="$(date -d @"${gravity_updated_raw}")" log_write " Last gravity run finished at: ${COL_CYAN}${gravity_updated}${COL_NC}" log_write "" @@ -1163,7 +1163,7 @@ analyze_gravity_list() { OLD_IFS="$IFS" IFS=$'\r\n' local gravity_sample=() - mapfile -t gravity_sample < <(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") + mapfile -t gravity_sample < <(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") log_write " ${COL_CYAN}----- First 10 Gravity Domains -----${COL_NC}" for line in "${gravity_sample[@]}"; do @@ -1195,7 +1195,7 @@ database_integrity_check(){ log_write "${INFO} Checking foreign key constraints of ${database} ... (this can take several minutes)" unset result - result="$(pihole-FTL sqlite3 "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)" + result="$(pihole-FTL sqlite3 -ni "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)" if [[ -z ${result} ]]; then log_write "${TICK} No foreign key errors in ${database}" else diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 14542e4b..4d97fec5 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -63,7 +63,7 @@ else fi fi # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) - deleted=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") + deleted=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") # Restart pihole-FTL to force reloading history sudo pihole restartdns diff --git a/gravity.sh b/gravity.sh index 8dc8edd8..f51103ff 100755 --- a/gravity.sh +++ b/gravity.sh @@ -48,7 +48,6 @@ gravityDBcopy="${piholeGitDir}/advanced/Templates/gravity_copy.sql" domainsExtension="domains" curl_connect_timeout=10 - # Check gravity temp directory if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then echo -e " ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}" @@ -64,7 +63,7 @@ gravityOLDfile="${gravityDIR}/gravity_old.db" # Generate new SQLite3 file from schema template generate_gravity_database() { - if ! pihole-FTL sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then + if ! pihole-FTL sqlite3 -ni "${gravityDBfile}" < "${gravityDBschema}"; then echo -e " ${CROSS} Unable to create ${gravityDBfile}" return 1 fi @@ -79,7 +78,7 @@ gravity_build_tree() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -118,7 +117,7 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -163,7 +162,7 @@ database_table_from_file() { # Get MAX(id) from domainlist when INSERTing into this table if [[ "${table}" == "domainlist" ]]; then - rowid="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" + rowid="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" if [[ -z "$rowid" ]]; then rowid=0 fi @@ -193,7 +192,7 @@ database_table_from_file() { # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -213,7 +212,7 @@ database_table_from_file() { # Check if a column with name ${2} exists in gravity table with name ${1} gravity_column_exists() { - output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) if [[ "${output}" == "1" ]]; then return 0 # Bash 0 is success fi @@ -228,7 +227,7 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -244,7 +243,7 @@ database_adlist_status() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -362,9 +361,9 @@ gravity_DownloadBlocklists() { # Retrieve source URLs from gravity database # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" - mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" - mapfile -t sourceTypes <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2> /dev/null)" + mapfile -t sources <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceTypes <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( @@ -393,7 +392,7 @@ gravity_DownloadBlocklists() { str="Preparing new gravity database" echo -ne " ${INFO} ${str}..." rm "${gravityTEMPfile}" > /dev/null 2>&1 - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -413,7 +412,7 @@ gravity_DownloadBlocklists() { copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}" fi - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -663,12 +662,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" + unique="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - pihole-FTL sqlite3 "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi @@ -749,7 +748,7 @@ database_recovery() { local str="Checking integrity of existing gravity database (this can take a while)" local option="${1}" echo -ne " ${INFO} ${str}..." - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)" + result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)" if [[ ${result} = "ok" ]]; then echo -e "${OVER} ${TICK} ${str} - no errors found" @@ -757,7 +756,7 @@ database_recovery() { str="Checking foreign keys of existing gravity database (this can take a while)" echo -ne " ${INFO} ${str}..." unset result - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)" + result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)" if [[ -z ${result} ]]; then echo -e "${OVER} ${TICK} ${str} - no errors found" if [[ "${option}" != "force" ]]; then @@ -776,7 +775,7 @@ database_recovery() { echo -ne " ${INFO} ${str}..." # We have to remove any possibly existing recovery database or this will fail rm -f "${gravityDBfile}.recovered" > /dev/null 2>&1 - if result="$(pihole-FTL sqlite3 "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 "${gravityDBfile}.recovered" 2>&1)"; then + if result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 -ni "${gravityDBfile}.recovered" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - success" mv "${gravityDBfile}" "${gravityDBfile}.old" mv "${gravityDBfile}.recovered" "${gravityDBfile}" From 3f7413d538dae3524b0a5d627668aecd60bcd815 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 9 Dec 2023 23:07:35 +0100 Subject: [PATCH 278/462] Add "-ni" to all sqlite3 invocations Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 30 ++++++++-------- advanced/Scripts/list.sh | 20 +++++------ advanced/Scripts/piholeARPTable.sh | 4 +-- advanced/Scripts/piholeDebug.sh | 12 +++---- advanced/Scripts/piholeLogFlush.sh | 2 +- advanced/Scripts/query.sh | 6 ++-- advanced/Scripts/webpage.sh | 12 +++---- gravity.sh | 36 +++++++++---------- 8 files changed, 61 insertions(+), 61 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index a7ba60a9..1459ecd9 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -19,13 +19,13 @@ upgrade_gravityDB(){ auditFile="${piholeDir}/auditlog.list" # Get database version - version="$(pihole-FTL sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" + version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" if [[ "$version" == "1" ]]; then # This migration script upgrades the gravity.db file by # adding the domain_audit table echo -e " ${INFO} Upgrading gravity database from version 1 to 2" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/1_to_2.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql" version=2 # Store audit domains in database table @@ -40,28 +40,28 @@ upgrade_gravityDB(){ # renaming the regex table to regex_blacklist, and # creating a new regex_whitelist table + corresponding linking table and views echo -e " ${INFO} Upgrading gravity database from version 2 to 3" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/2_to_3.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql" version=3 fi if [[ "$version" == "3" ]]; then # This migration script unifies the formally separated domain # lists into a single table with a UNIQUE domain constraint echo -e " ${INFO} Upgrading gravity database from version 3 to 4" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/3_to_4.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql" version=4 fi if [[ "$version" == "4" ]]; then # This migration script upgrades the gravity and list views # implementing necessary changes for per-client blocking echo -e " ${INFO} Upgrading gravity database from version 4 to 5" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/4_to_5.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql" version=5 fi if [[ "$version" == "5" ]]; then # This migration script upgrades the adlist view # to return an ID used in gravity.sh echo -e " ${INFO} Upgrading gravity database from version 5 to 6" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/5_to_6.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql" version=6 fi if [[ "$version" == "6" ]]; then @@ -69,7 +69,7 @@ upgrade_gravityDB(){ # which is automatically associated to all clients not # having their own group assignments echo -e " ${INFO} Upgrading gravity database from version 6 to 7" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/6_to_7.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql" version=7 fi if [[ "$version" == "7" ]]; then @@ -77,21 +77,21 @@ upgrade_gravityDB(){ # to ensure uniqueness on the group name # We also add date_added and date_modified columns echo -e " ${INFO} Upgrading gravity database from version 7 to 8" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/7_to_8.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql" version=8 fi if [[ "$version" == "8" ]]; then # This migration fixes some issues that were introduced # in the previous migration script. echo -e " ${INFO} Upgrading gravity database from version 8 to 9" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/8_to_9.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql" version=9 fi if [[ "$version" == "9" ]]; then # This migration drops unused tables and creates triggers to remove # obsolete groups assignments when the linked items are deleted echo -e " ${INFO} Upgrading gravity database from version 9 to 10" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/9_to_10.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql" version=10 fi if [[ "$version" == "10" ]]; then @@ -101,31 +101,31 @@ upgrade_gravityDB(){ # to keep the copying process generic (needs the same columns in both the # source and the destination databases). echo -e " ${INFO} Upgrading gravity database from version 10 to 11" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/10_to_11.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql" version=11 fi if [[ "$version" == "11" ]]; then # Rename group 0 from "Unassociated" to "Default" echo -e " ${INFO} Upgrading gravity database from version 11 to 12" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/11_to_12.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql" version=12 fi if [[ "$version" == "12" ]]; then # Add column date_updated to adlist table echo -e " ${INFO} Upgrading gravity database from version 12 to 13" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/12_to_13.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql" version=13 fi if [[ "$version" == "13" ]]; then # Add columns number and status to adlist table echo -e " ${INFO} Upgrading gravity database from version 13 to 14" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/13_to_14.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql" version=14 fi if [[ "$version" == "14" ]]; then # Changes the vw_adlist created in 5_to_6 echo -e " ${INFO} Upgrading gravity database from version 14 to 15" - pihole-FTL sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql" version=15 fi } diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index b76a7ef7..76558e58 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -150,18 +150,18 @@ AddDomain() { domain="$1" # Is the domain in the list we want to add it to? - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" requestedListname="$(GetListnameFromTypeId "${typeId}")" if [[ "${num}" -ne 0 ]]; then - existingTypeId="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" + existingTypeId="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" if [[ "${existingTypeId}" == "${typeId}" ]]; then if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${requestedListname}, no need to add!" fi else existingListname="$(GetListnameFromTypeId "${existingTypeId}")" - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" if [[ "${verbose}" == true ]]; then echo -e " ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!" fi @@ -177,10 +177,10 @@ AddDomain() { # Insert only the domain here. The enabled and date_added fields will be filled # with their default values (enabled = true, date_added = current timestamp) if [[ -z "${comment}" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" else # also add comment when variable has been set through the "--comment" option - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" fi } @@ -189,7 +189,7 @@ RemoveDomain() { domain="$1" # Is the domain in the list we want to remove it from? - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" requestedListname="$(GetListnameFromTypeId "${typeId}")" @@ -206,14 +206,14 @@ RemoveDomain() { fi reload=true # Remove it from the current list - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" } Displaylist() { local count num_pipes domain enabled status nicedate requestedListname requestedListname="$(GetListnameFromTypeId "${typeId}")" - data="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" + data="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" if [[ -z $data ]]; then echo -e "Not showing empty list" @@ -251,10 +251,10 @@ Displaylist() { } NukeList() { - count=$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") + count=$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") listname="$(GetListnameFromTypeId "${typeId}")" if [ "$count" -gt 0 ];then - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" echo " ${TICK} Removed ${count} domain(s) from the ${listname}" else echo " ${INFO} ${listname} already empty. Nothing to do!" diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index 5daa025d..b92dd124 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -39,7 +39,7 @@ flushARP(){ # Truncate network_addresses table in pihole-FTL.db # This needs to be done before we can truncate the network table due to # foreign key constraints - if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then + if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network_addresses table" echo " Database location: ${DBFILE}" echo " Output: ${output}" @@ -47,7 +47,7 @@ flushARP(){ fi # Truncate network table in pihole-FTL.db - if ! output=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then + if ! output=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM network" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network table" echo " Database location: ${DBFILE}" echo " Output: ${output}" diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 1ca52f45..fb1efbeb 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -865,7 +865,7 @@ dig_at() { # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") + random_url=$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity WHERE domain not like '||%^' ORDER BY RANDOM() LIMIT 1") # Fallback if no non-ABP style domains were found if [ -z "${random_url}" ]; then random_url="flurry.com" @@ -1226,7 +1226,7 @@ show_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \ + pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1251,7 +1251,7 @@ show_FTL_db_entries() { IFS=$'\r\n' local entries=() mapfile -t entries < <(\ - pihole-FTL sqlite3 "${PIHOLE_FTL_DB_FILE}" \ + pihole-FTL sqlite3 -ni "${PIHOLE_FTL_DB_FILE}" \ -cmd ".headers on" \ -cmd ".mode column" \ -cmd ".width ${widths}" \ @@ -1317,7 +1317,7 @@ analyze_gravity_list() { fi show_db_entries "Info table" "SELECT property,value FROM info" "20 40" - gravity_updated_raw="$(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" + gravity_updated_raw="$(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT value FROM info where property = 'updated'")" gravity_updated="$(date -d @"${gravity_updated_raw}")" log_write " Last gravity run finished at: ${COL_CYAN}${gravity_updated}${COL_NC}" log_write "" @@ -1325,7 +1325,7 @@ analyze_gravity_list() { OLD_IFS="$IFS" IFS=$'\r\n' local gravity_sample=() - mapfile -t gravity_sample < <(pihole-FTL sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") + mapfile -t gravity_sample < <(pihole-FTL sqlite3 -ni "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") log_write " ${COL_CYAN}----- First 10 Gravity Domains -----${COL_NC}" for line in "${gravity_sample[@]}"; do @@ -1357,7 +1357,7 @@ database_integrity_check(){ log_write "${INFO} Checking foreign key constraints of ${database} ... (this can take several minutes)" unset result - result="$(pihole-FTL sqlite3 "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)" + result="$(pihole-FTL sqlite3 -ni "${database}" -cmd ".headers on" -cmd ".mode column" "PRAGMA foreign_key_check" 2>&1 & spinner)" if [[ -z ${result} ]]; then log_write "${TICK} No foreign key errors in ${database}" else diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 3473fad5..b06aac8b 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -63,7 +63,7 @@ else fi fi # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) - deleted=$(pihole-FTL sqlite3 "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") + deleted=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") # Restart pihole-FTL to force reloading history sudo pihole restartdns diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 1d3b0a29..ebcc6f79 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -96,7 +96,7 @@ scanDatabaseTable() { # Are there ABP entries on gravity? # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set abpquerystr="SELECT EXISTS (SELECT 1 FROM info WHERE property='abp_domains' and value='1')" - abpfound="$(pihole-FTL sqlite3 "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null + abpfound="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null # Create search string for ABP entries only if needed if [ "${abpfound}" -eq 1 ]; then @@ -129,7 +129,7 @@ scanDatabaseTable() { fi # Send prepared query to gravity database - result="$(pihole-FTL sqlite3 -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null + result="$(pihole-FTL sqlite3 -ni -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null if [[ -z "${result}" ]]; then # Return early when there are no matches in this table return @@ -166,7 +166,7 @@ scanRegexDatabaseTable() { list_type="${3:-}" # Query all regex from the corresponding database tables - mapfile -t regexList < <(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${list_type}" 2> /dev/null) + mapfile -t regexList < <(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain FROM domainlist WHERE type = ${list_type}" 2> /dev/null) # If we have regexps to process if [[ "${#regexList[@]}" -ne 0 ]]; then diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index e05a6aff..62ab4ea9 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -560,13 +560,13 @@ CustomizeAdLists() { if CheckUrl "${address}"; then if [[ "${args[2]}" == "enable" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'" elif [[ "${args[2]}" == "disable" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'" elif [[ "${args[2]}" == "add" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')" elif [[ "${args[2]}" == "del" ]]; then - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'" else echo "Not permitted" return 1 @@ -700,12 +700,12 @@ addAudit() done # Insert only the domain here. The date_added field will be # filled with its default value (date_added = current timestamp) - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};" } clearAudit() { - pihole-FTL sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domain_audit;" } SetPrivacyLevel() { diff --git a/gravity.sh b/gravity.sh index db199090..62631e3c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -84,7 +84,7 @@ fi # Generate new SQLite3 file from schema template generate_gravity_database() { - if ! pihole-FTL sqlite3 "${gravityDBfile}" < "${gravityDBschema}"; then + if ! pihole-FTL sqlite3 -ni "${gravityDBfile}" < "${gravityDBschema}"; then echo -e " ${CROSS} Unable to create ${gravityDBfile}" return 1 fi @@ -99,7 +99,7 @@ gravity_swap_databases() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -135,7 +135,7 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -179,7 +179,7 @@ database_table_from_file() { # Get MAX(id) from domainlist when INSERTing into this table if [[ "${table}" == "domainlist" ]]; then - rowid="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" + rowid="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT MAX(id) FROM domainlist;")" if [[ -z "$rowid" ]]; then rowid=0 fi @@ -209,7 +209,7 @@ database_table_from_file() { # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\n.mode csv\\n.import \"%s\" %s\\n" "${tmpFile}" "${table}" | pihole-FTL sqlite3 -ni "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -229,7 +229,7 @@ database_table_from_file() { # Check if a column with name ${2} exists in gravity table with name ${1} gravity_column_exists() { - output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) if [[ "${output}" == "1" ]]; then return 0 # Bash 0 is success fi @@ -244,7 +244,7 @@ database_adlist_number() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${2}" "${3}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -260,7 +260,7 @@ database_adlist_status() { return; fi - output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 "${gravityTEMPfile}"; } 2>&1 ) + output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -378,8 +378,8 @@ gravity_DownloadBlocklists() { # Retrieve source URLs from gravity database # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" - mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" + mapfile -t sources <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" + mapfile -t sourceIDs <<< "$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( @@ -408,7 +408,7 @@ gravity_DownloadBlocklists() { str="Preparing new gravity database" echo -ne " ${INFO} ${str}..." rm "${gravityTEMPfile}" > /dev/null 2>&1 - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" < "${gravityDBschema}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -428,7 +428,7 @@ gravity_DownloadBlocklists() { copyGravity="${copyGravity//"${gravityDBfile_default}"/"${gravityDBfile}"}" fi - output=$( { pihole-FTL sqlite3 "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" <<< "${copyGravity}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -668,12 +668,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" + unique="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi @@ -754,7 +754,7 @@ database_recovery() { local str="Checking integrity of existing gravity database (this can take a while)" local option="${1}" echo -ne " ${INFO} ${str}..." - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)" + result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA integrity_check" 2>&1)" if [[ ${result} = "ok" ]]; then echo -e "${OVER} ${TICK} ${str} - no errors found" @@ -762,7 +762,7 @@ database_recovery() { str="Checking foreign keys of existing gravity database (this can take a while)" echo -ne " ${INFO} ${str}..." unset result - result="$(pihole-FTL sqlite3 "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)" + result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "PRAGMA foreign_key_check" 2>&1)" if [[ -z ${result} ]]; then echo -e "${OVER} ${TICK} ${str} - no errors found" if [[ "${option}" != "force" ]]; then @@ -781,7 +781,7 @@ database_recovery() { echo -ne " ${INFO} ${str}..." # We have to remove any possibly existing recovery database or this will fail rm -f "${gravityDBfile}.recovered" > /dev/null 2>&1 - if result="$(pihole-FTL sqlite3 "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 "${gravityDBfile}.recovered" 2>&1)"; then + if result="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" ".recover" | pihole-FTL sqlite3 -ni "${gravityDBfile}.recovered" 2>&1)"; then echo -e "${OVER} ${TICK} ${str} - success" mv "${gravityDBfile}" "${gravityDBfile}.old" mv "${gravityDBfile}.recovered" "${gravityDBfile}" From fe4d934a406cf4fa45a72668170868bd57448537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 22:31:36 +0100 Subject: [PATCH 279/462] Simplify pihole -v MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/version.sh | 138 ++++++++---------------------------- pihole | 5 +- 2 files changed, 31 insertions(+), 112 deletions(-) diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index 2983c04e..ff7f0815 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -8,6 +8,10 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. +# Ignore warning about `local` being undefinded in POSIX +# shellcheck disable=SC3043 +# https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions + # Source the versions file poupulated by updatechecker.sh cachedVersions="/etc/pihole/versions" @@ -21,118 +25,34 @@ else . "$cachedVersions" fi -getLocalVersion() { - case ${1} in - "Pi-hole" ) echo "${CORE_VERSION:=N/A}";; - "web" ) echo "${WEB_VERSION:=N/A}";; - "FTL" ) echo "${FTL_VERSION:=N/A}";; - esac -} +main() { + local details + details=$1 -getLocalHash() { - case ${1} in - "Pi-hole" ) echo "${CORE_HASH:=N/A}";; - "web" ) echo "${WEB_HASH:=N/A}";; - "FTL" ) echo "${FTL_HASH:=N/A}";; - esac -} - -getRemoteHash(){ - case ${1} in - "Pi-hole" ) echo "${GITHUB_CORE_HASH:=N/A}";; - "web" ) echo "${GITHUB_WEB_HASH:=N/A}";; - "FTL" ) echo "${GITHUB_FTL_HASH:=N/A}";; - esac -} - -getRemoteVersion(){ - case ${1} in - "Pi-hole" ) echo "${GITHUB_CORE_VERSION:=N/A}";; - "web" ) echo "${GITHUB_WEB_VERSION:=N/A}";; - "FTL" ) echo "${GITHUB_FTL_VERSION:=N/A}";; - esac -} - -getLocalBranch(){ - case ${1} in - "Pi-hole" ) echo "${CORE_BRANCH:=N/A}";; - "web" ) echo "${WEB_BRANCH:=N/A}";; - "FTL" ) echo "${FTL_BRANCH:=N/A}";; - esac -} - -versionOutput() { - - [ "$2" = "-c" ] || [ "$2" = "--current" ] || [ -z "$2" ] && current=$(getLocalVersion "${1}") && branch=$(getLocalBranch "${1}") - [ "$2" = "-l" ] || [ "$2" = "--latest" ] || [ -z "$2" ] && latest=$(getRemoteVersion "${1}") - if [ "$2" = "--hash" ]; then - [ "$3" = "-c" ] || [ "$3" = "--current" ] || [ -z "$3" ] && curHash=$(getLocalHash "${1}") && branch=$(getLocalBranch "${1}") - [ "$3" = "-l" ] || [ "$3" = "--latest" ] || [ -z "$3" ] && latHash=$(getRemoteHash "${1}") && branch=$(getLocalBranch "${1}") - fi - - # We do not want to show the branch name when we are on master, - # blank out the variable in this case - if [ "$branch" = "master" ]; then - branch="" + if [ "${details}" = true ]; then + echo "Core" + echo " Version is ${CORE_VERSION:=N/A} (Latest: ${GITHUB_CORE_VERSION:=N/A})" + echo " Branch is ${CORE_BRANCH:=N/A}" + echo " Hash is ${CORE_HASH:=N/A} (Latest: ${GITHUB_CORE_HASH:=N/A})" + echo "Web" + echo " Version is ${WEB_VERSION:=N/A} (Latest: ${GITHUB_WEB_VERSION:=N/A})" + echo " Branch is ${WEB_BRANCH:=N/A}" + echo " Hash is ${WEB_HASH:=N/A} (Latest: ${GITHUB_WEB_HASH:=N/A})" + echo "FTL" + echo " Version is ${FTL_VERSION:=N/A} (Latest: ${GITHUB_FTL_VERSION:=N/A})" + echo " Branch is ${FTL_BRANCH:=N/A}" + echo " Hash is ${FTL_HASH:=N/A} (Latest: ${GITHUB_FTL_HASH:=N/A})" else - branch="$branch " + echo "Core version is ${CORE_VERSION:=N/A} (Latest: ${GITHUB_CORE_VERSION:=N/A})" + echo "Web version is ${WEB_VERSION:=N/A} (Latest: ${GITHUB_WEB_VERSION:=N/A})" + echo "FTL version is ${FTL_VERSION:=N/A} (Latest: ${GITHUB_FTL_VERSION:=N/A})" fi - - if [ -n "$current" ] && [ -n "$latest" ]; then - output="${1} version is $branch$current (Latest: $latest)" - elif [ -n "$current" ] && [ -z "$latest" ]; then - output="Current ${1} version is $branch$current" - elif [ -z "$current" ] && [ -n "$latest" ]; then - output="Latest ${1} version is $latest" - elif [ -n "$curHash" ] && [ -n "$latHash" ]; then - output="Local ${1} hash is $curHash (Remote: $latHash)" - elif [ -n "$curHash" ] && [ -z "$latHash" ]; then - output="Current local ${1} hash is $curHash" - elif [ -z "$curHash" ] && [ -n "$latHash" ]; then - output="Latest remote ${1} hash is $latHash" - elif [ -z "$curHash" ] && [ -z "$latHash" ]; then - output="Hashes for ${1} not available" - else - errorOutput - return 1 - fi - - [ -n "$output" ] && echo " $output" } -errorOutput() { - echo " Invalid Option! Try 'pihole -v --help' for more information." - exit 1 -} - -defaultOutput() { - versionOutput "Pi-hole" "$@" - versionOutput "web" "$@" - versionOutput "FTL" "$@" -} - -helpFunc() { - echo "Usage: pihole -v [repo | option] [option] -Example: 'pihole -v -p -l' -Show Pi-hole, Web Console & FTL versions - -Repositories: - -p, --pihole Only retrieve info regarding Pi-hole repository - -w, --web Only retrieve info regarding web repository - -f, --ftl Only retrieve info regarding FTL repository - -Options: - -c, --current Return the current version - -l, --latest Return the latest version - --hash Return the GitHub hash from your local repositories - -h, --help Show this help dialog" - exit 0 -} - -case "${1}" in - "-p" | "--pihole" ) shift; versionOutput "Pi-hole" "$@";; - "-w" | "--web" ) shift; versionOutput "web" "$@";; - "-f" | "--ftl" ) shift; versionOutput "FTL" "$@";; - "-h" | "--help" ) helpFunc;; - * ) defaultOutput "$@";; +# Process options (if present) +case "$1" in + "-vv" ) details=true;; + * ) details=false;; esac + +main "${details}" diff --git a/pihole b/pihole index 47da4ddd..ef93146c 100755 --- a/pihole +++ b/pihole @@ -140,7 +140,6 @@ uninstallFunc() { } versionFunc() { - shift exec "${PI_HOLE_SCRIPT_DIR}"/version.sh "$@" } @@ -508,7 +507,7 @@ Options: -up, updatePihole Update Pi-hole subsystems Add '--check-only' to exit script before update is performed. -v, version Show installed versions of Pi-hole, Web Interface & FTL - Add '-h' for more info on version usage + Use -vv for detailed information. uninstall Uninstall Pi-hole from your system status Display the running status of Pi-hole subsystems enable Enable Pi-hole subsystems @@ -531,7 +530,7 @@ fi need_root=1 case "${1}" in "-h" | "help" | "--help" ) helpFunc;; - "-v" | "version" ) versionFunc "$@";; + "-v" | "-vv" | "version" ) versionFunc "$@";; "-c" | "chronometer" ) chronometerFunc "$@";; "-q" | "query" ) queryFunc "$@";; "status" ) statusFunc "$2";; From b333e30162a07287bcf313ec9da01e1644c72d52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 22:55:01 +0100 Subject: [PATCH 280/462] Remove version options from bash completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/bash-completion/pihole | 4 ---- 1 file changed, 4 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 4fe8f83a..89e02d2f 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -34,10 +34,6 @@ _pihole() { opts_update="--check-only" COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) ;; - "version") - opts_version="\--web \--current \--ftl \--hash \--latest \--pihole" - COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) - ;; "core"|"admin"|"ftl") if [[ "$prev2" == "checkout" ]]; then opts_checkout="master dev" From c3c31a1a60dcf68ae3b08a2c474485fdbd3c9cb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 9 Dec 2023 23:09:58 +0100 Subject: [PATCH 281/462] Print version details automatically if not on master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/version.sh | 16 ++++++++-------- pihole | 5 ++--- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index ff7f0815..2422ea6d 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -27,7 +27,13 @@ fi main() { local details - details=$1 + details=false + + # Automatically show detailed information if + # at least one of the components is not on master branch + if [ ! "${CORE_BRANCH}" = "master" ] || [ ! "${WEB_BRANCH}" = "master" ] || [ ! "${FTL_BRANCH}" = "master" ] ; then + details=true + fi if [ "${details}" = true ]; then echo "Core" @@ -49,10 +55,4 @@ main() { fi } -# Process options (if present) -case "$1" in - "-vv" ) details=true;; - * ) details=false;; -esac - -main "${details}" +main diff --git a/pihole b/pihole index ef93146c..7c84771c 100755 --- a/pihole +++ b/pihole @@ -140,7 +140,7 @@ uninstallFunc() { } versionFunc() { - exec "${PI_HOLE_SCRIPT_DIR}"/version.sh "$@" + exec "${PI_HOLE_SCRIPT_DIR}"/version.sh } restartDNS() { @@ -507,7 +507,6 @@ Options: -up, updatePihole Update Pi-hole subsystems Add '--check-only' to exit script before update is performed. -v, version Show installed versions of Pi-hole, Web Interface & FTL - Use -vv for detailed information. uninstall Uninstall Pi-hole from your system status Display the running status of Pi-hole subsystems enable Enable Pi-hole subsystems @@ -530,7 +529,7 @@ fi need_root=1 case "${1}" in "-h" | "help" | "--help" ) helpFunc;; - "-v" | "-vv" | "version" ) versionFunc "$@";; + "-v" | "version" ) versionFunc;; "-c" | "chronometer" ) chronometerFunc "$@";; "-q" | "query" ) queryFunc "$@";; "status" ) statusFunc "$2";; From e8338d059009f009f8b93461e4fee51792acb600 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 10 Dec 2023 10:41:00 +0100 Subject: [PATCH 282/462] Install FTL's development branch to get the latest features during tests Signed-off-by: DL6ER --- test/test_any_automated_install.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 840d1df7..c1b91664 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -176,6 +176,12 @@ def test_installPihole_fresh_install_readableFiles(host): setup_var_file += "INSTALL_WEB_INTERFACE=true\n" setup_var_file += "EOF\n" host.run(setup_var_file) + # Install FTL's development branch to get the latest features + host.run( + """ + echo "development" > /etc/pihole/ftlbranch + """ + ) install = host.run( """ export TERM=xterm @@ -431,6 +437,12 @@ def test_installPihole_fresh_install_readableBlockpage(host, test_webpage): setup_var_file += "INSTALL_WEB_INTERFACE=true\n" setup_var_file += "EOF\n" host.run(setup_var_file) + # Install FTL's development branch to get the latest features + host.run( + """ + echo "development" > /etc/pihole/ftlbranch + """ + ) installWeb = host.run( """ export TERM=xterm From 0cfcdc4b50e16612bf7572f290c1960c0f39e873 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Dec 2023 10:08:32 +0000 Subject: [PATCH 283/462] Bump github/codeql-action from 2 to 3 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6544db61..fc821194 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -29,12 +29,12 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: 'python' - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From c6049d400214d45f6bd9229e6fd3291a3d53f3dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Dec 2023 10:30:20 +0000 Subject: [PATCH 284/462] Bump github/codeql-action from 2 to 3 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 6544db61..fc821194 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -29,12 +29,12 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: 'python' - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From 0066c6fbffa889525751478468b1cfba77272e20 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 05:29:11 +0100 Subject: [PATCH 285/462] Make IDs of anti-/gravity lists available in vw_(anti)gravity Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 6 +++++ .../database_migration/gravity/17_to_18.sql | 25 +++++++++++++++++++ advanced/Templates/gravity.db.sql | 6 ++--- gravity.sh | 2 +- 4 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/17_to_18.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 7c4deaa7..b263b40d 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -141,4 +141,10 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql" version=17 fi + if [[ "$version" == "17" ]]; then + # Add adlist.id to vw_gravity and vw_antigravity + echo -e " ${INFO} Upgrading gravity database from version 17 to 18" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql" + version=18 + fi } diff --git a/advanced/Scripts/database_migration/gravity/17_to_18.sql b/advanced/Scripts/database_migration/gravity/17_to_18.sql new file mode 100644 index 00000000..00171a9a --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/17_to_18.sql @@ -0,0 +1,25 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +DROP VIEW vw_gravity; +CREATE VIEW vw_gravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id + FROM gravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id + LEFT JOIN adlist ON adlist.id = gravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); + +DROP VIEW vw_antigravity; +CREATE VIEW vw_antigravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id + FROM antigravity + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id + LEFT JOIN adlist ON adlist.id = antigravity.adlist_id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) AND adlist.type = 1; + +UPDATE info SET value = 18 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 46f26ba7..097b0a78 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -65,7 +65,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','17'); +INSERT INTO "info" VALUES('version','18'); CREATE TABLE domain_audit ( @@ -144,14 +144,14 @@ CREATE VIEW vw_regex_blacklist AS SELECT domain, domainlist.id AS id, domainlist AND domainlist.type = 3 ORDER BY domainlist.id; -CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id +CREATE VIEW vw_gravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id FROM gravity LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id LEFT JOIN adlist ON adlist.id = gravity.adlist_id LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); -CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id +CREATE VIEW vw_antigravity AS SELECT domain, adlist.id AS adlist_id, adlist_by_group.group_id AS group_id FROM antigravity LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id LEFT JOIN adlist ON adlist.id = antigravity.adlist_id diff --git a/gravity.sh b/gravity.sh index f51103ff..ebe7e740 100755 --- a/gravity.sh +++ b/gravity.sh @@ -78,7 +78,7 @@ gravity_build_tree() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) + output=$( { time pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } ) status="$?" if [[ "${status}" -ne 0 ]]; then From 8e8c7ecad212d45548dc89f408d2545d332a37a5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 05:41:43 +0100 Subject: [PATCH 286/462] Run gravity upgrade on checkout/update Signed-off-by: DL6ER --- automated install/basic-install.sh | 5 +++++ gravity.sh | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f5bf15e6..5863e818 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2225,6 +2225,11 @@ main() { # but before starting or restarting the ftl service disable_resolved_stublistener + # Check if gravity database needs to be upgraded. If so, do it without rebuilding + # gravity altogether. This may be a very long running task needlessly blocking + # the update process. + /opt/pihole/gravity.sh --upgrade + printf " %b Restarting services...\\n" "${INFO}" # Start services diff --git a/gravity.sh b/gravity.sh index ebe7e740..216a67a5 100755 --- a/gravity.sh +++ b/gravity.sh @@ -78,7 +78,7 @@ gravity_build_tree() { echo -ne " ${INFO} ${str}..." # The index is intentionally not UNIQUE as poor quality adlists may contain domains more than once - output=$( { time pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } ) + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "CREATE INDEX idx_gravity ON gravity (domain, adlist_id);"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -831,6 +831,7 @@ for var in "$@"; do case "${var}" in "-f" | "--force" ) forceDelete=true;; "-r" | "--repair" ) repairSelector "$3";; + "-u" | "--upgrade" ) upgrade_gravityDB "${gravityDBfile}" "${piholeDir}"; exit 0;; "-h" | "--help" ) helpFunc;; esac done From 1b87ec067ef8d19df0cc6af9f4aa30ba69696d3c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 06:12:18 +0100 Subject: [PATCH 287/462] Exit early if the database does not exist (e.g. in some CI tests) Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index b263b40d..7b356b3a 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -18,6 +18,11 @@ upgrade_gravityDB(){ piholeDir="${2}" auditFile="${piholeDir}/auditlog.list" + # Exit early if the database does not exist (e.g. in CI tests) + if [[ ! -f "${database}" ]]; then + return + fi + # Get database version version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" From 0c6363572b2ddde8c44f3bd5d06dfff92f929380 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Dec 2023 22:00:49 +0100 Subject: [PATCH 288/462] Append the interface to the gateway address if it is a link-local address Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index aa0e61c4..a78f94ee 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -547,17 +547,24 @@ ping_gateway() { ping_ipv4_or_ipv6 "${protocol}" # Check if we are using IPv4 or IPv6 # Find the default gateways using IPv4 or IPv6 - local gateway + local gateway gateway_addr gateway_iface log_write "${INFO} Default IPv${protocol} gateway(s):" while IFS= read -r gateway; do - log_write " ${gateway}" - done < <(ip -"${protocol}" route | grep default | cut -d ' ' -f 3) + log_write " $(cut -d ' ' -f 3 <<< "${gateway}")%$(cut -d ' ' -f 5 <<< "${gateway}")" + done < <(ip -"${protocol}" route | grep default) - gateway=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 3 | head -n 1) + gateway_addr=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 3 | head -n 1) + gateway_iface=$(ip -"${protocol}" route | grep default | cut -d ' ' -f 5 | head -n 1) # If there was at least one gateway - if [ -n "${gateway}" ]; then + if [ -n "${gateway_addr}" ]; then + # Append the interface to the gateway address if it is a link-local address + if [[ "${gateway_addr}" =~ ^fe80 ]]; then + gateway="${gateway_addr}%${gateway_iface}" + else + gateway="${gateway_addr}" + fi # Let the user know we will ping the gateway for a response log_write " * Pinging first gateway ${gateway}..." # Try to quietly ping the gateway 3 times, with a timeout of 3 seconds, using numeric output only, From 00340136bdc517cc71d4458fb5ccd8d784724767 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 26 Dec 2023 21:35:24 +0100 Subject: [PATCH 289/462] Extend %iface logic to the dig test. Also fix intentation in this function Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 39 +++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index a78f94ee..2c3ebb14 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -764,24 +764,29 @@ dig_at() { # Removes CIDR and everything thereafter (e.g., scope properties) addresses="$(ip address show dev "${iface}" | sed "/${sed_selector} /!d;s/^.*${sed_selector} //g;s/\/.*$//g;")" if [ -n "${addresses}" ]; then - while IFS= read -r local_address ; do + while IFS= read -r local_address ; do + # If ${local_address} is an IPv6 link-local address, append the interface name to it + if [[ "${local_address}" =~ ^fe80 ]]; then + local_address="${local_address}%${iface}" + fi + # Check if Pi-hole can use itself to block a domain - if local_dig="$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" "${record_type}")"; then - # If it can, show success - if [[ "${local_dig}" == *"status: NOERROR"* ]]; then - local_dig="NOERROR" - elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then - local_dig="NXDOMAIN" - else - # Extract the first entry in the answer section from dig's output, - # replacing any multiple spaces and tabs with a single space - local_dig="$(echo "${local_dig}" | grep -A1 "ANSWER SECTION" | grep -v "ANSWER SECTION" | tr -s " \t" " ")" - fi - log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" - else - # Otherwise, show a failure - log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} on ${COL_RED}${iface}${COL_NC} (${COL_RED}${local_address}${COL_NC})" - fi + if local_dig="$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @"${local_address}" "${record_type}")"; then + # If it can, show success + if [[ "${local_dig}" == *"status: NOERROR"* ]]; then + local_dig="NOERROR" + elif [[ "${local_dig}" == *"status: NXDOMAIN"* ]]; then + local_dig="NXDOMAIN" + else + # Extract the first entry in the answer section from dig's output, + # replacing any multiple spaces and tabs with a single space + local_dig="$(echo "${local_dig}" | grep -A1 "ANSWER SECTION" | grep -v "ANSWER SECTION" | tr -s " \t" " ")" + fi + log_write "${TICK} ${random_url} ${COL_GREEN}is ${local_dig}${COL_NC} on ${COL_CYAN}${iface}${COL_NC} (${COL_CYAN}${local_address}${COL_NC})" + else + # Otherwise, show a failure + log_write "${CROSS} ${COL_RED}Failed to resolve${COL_NC} ${random_url} on ${COL_RED}${iface}${COL_NC} (${COL_RED}${local_address}${COL_NC})" + fi done <<< "${addresses}" else log_write "${TICK} No IPv${protocol} address available on ${COL_CYAN}${iface}${COL_NC}" From d2828310f240aed098e5b3d787f5d63f47575a7a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 28 Dec 2023 11:12:20 +0100 Subject: [PATCH 290/462] Modify DELETE triggers to delete BEFORE instead of AFTER to prevent possible foreign key constraint violations Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 7 +++++ .../database_migration/gravity/18_to_19.sql | 27 +++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 advanced/Scripts/database_migration/gravity/18_to_19.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 7b356b3a..e99f1df2 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -152,4 +152,11 @@ upgrade_gravityDB(){ pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql" version=18 fi + if [[ "$version" == "18" ]]; then + # Modify DELETE triggers to delete BEFORE instead of AFTER to prevent + # foreign key constraint violations + echo -e " ${INFO} Upgrading gravity database from version 18 to 19" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/18_to_19.sql" + version=19 + fi } diff --git a/advanced/Scripts/database_migration/gravity/18_to_19.sql b/advanced/Scripts/database_migration/gravity/18_to_19.sql new file mode 100644 index 00000000..c85a4d57 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/18_to_19.sql @@ -0,0 +1,27 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +DROP TRIGGER tr_domainlist_delete; +CREATE TRIGGER tr_domainlist_delete BEFORE DELETE ON domainlist + BEGIN + DELETE FROM domainlist_by_group WHERE domainlist_id = OLD.id; + END; + +DROP TRIGGER tr_adlist_delete; +CREATE TRIGGER tr_adlist_delete BEFORE DELETE ON adlist + BEGIN + DELETE FROM adlist_by_group WHERE adlist_id = OLD.id; + END; + +DROP TRIGGER tr_client_delete; +CREATE TRIGGER tr_client_delete BEFORE DELETE ON client + BEGIN + DELETE FROM client_by_group WHERE client_id = OLD.id; + END; + +UPDATE info SET value = 19 WHERE property = 'version'; + +COMMIT; From 9c3578856f1c47887d7af5f1bc972ce44f5316ec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jan 2024 10:02:39 +0000 Subject: [PATCH 291/462] Bump pytest from 7.4.3 to 7.4.4 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.3 to 7.4.4. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.3...7.4.4) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index bfc6d027..0873d097 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.3 +pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.11.4 From 05e7d0ee9263e17e0bc6706b3822310d7ed084a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jan 2024 10:10:22 +0000 Subject: [PATCH 292/462] Bump pytest from 7.4.3 to 7.4.4 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.3 to 7.4.4. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.3...7.4.4) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index bfc6d027..0873d097 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.3 +pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.11.4 From 220c0675ef67adea60161fdd38f409736bcaaac0 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 7 Jan 2024 21:03:48 +0000 Subject: [PATCH 293/462] As we do in other tests, specify `development-v6` branch of FTL is downloaded for the tests. We need to change this to `develoment` once v6 is released Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index c53070dc..ab301a6e 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -239,6 +239,7 @@ def test_FTL_detect_aarch64_no_errors(host): mock_command("uname", {"-m": ("aarch64", "0")}, host) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -272,6 +273,7 @@ def test_FTL_detect_armv6_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -305,6 +307,7 @@ def test_FTL_detect_armv7l_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -338,6 +341,7 @@ def test_FTL_detect_armv7_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -371,6 +375,7 @@ def test_FTL_detect_armv8a_no_errors(host): ) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -393,6 +398,7 @@ def test_FTL_detect_x86_64_no_errors(host): """ detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -415,6 +421,7 @@ def test_FTL_detect_unknown_no_errors(host): mock_command("uname", {"-m": ("mips", "0")}, host) detectPlatform = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -442,6 +449,7 @@ def test_FTL_download_aarch64_no_errors(host): ) download_binary = host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user FTLinstall "pihole-FTL-aarch64-linux-gnu" @@ -458,6 +466,7 @@ def test_FTL_development_binary_installed_and_responsive_no_errors(host): """ host.run( """ + echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) From df9c60e3511f7871c1392f19bebbef1112be64ac Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 8 Jan 2024 18:26:59 +0000 Subject: [PATCH 294/462] Reduce code duplication in FTL arch detect tests. Use parametrize to run the same test with different parameters for each arch we need to test. Also include detection of unusupported in this test. Create FTL_BRANCH constant to be used in functions (less places to remember to change it) Signed-off-by: Adam Warner --- test/test_any_automated_install.py | 253 +++++------------------------ 1 file changed, 37 insertions(+), 216 deletions(-) diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index ab301a6e..0930f0af 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -12,6 +12,8 @@ from .conftest import ( run_script, ) +FTL_BRANCH = "development-v6" + def test_supported_package_manager(host): """ @@ -80,11 +82,7 @@ def test_installPihole_fresh_install_readableFiles(host): host.run("command -v dnf > /dev/null && dnf install -y man") host.run("command -v yum > /dev/null && yum install -y man") # Workaround to get FTLv6 installed until it reaches master branch - host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - """ - ) + host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch') install = host.run( """ export TERM=xterm @@ -231,49 +229,37 @@ def test_update_package_cache_failure_no_errors(host): assert "Error: Unable to update package cache." in updateCache.stdout -def test_FTL_detect_aarch64_no_errors(host): +@pytest.mark.parametrize( + "arch,detected_string,supported", + [ + ("aarch64", "AArch64 (64 Bit ARM)", True), + ("armv6", "ARMv6", True), + ("armv7l", "ARMv7 (or newer)", True), + ("armv7", "ARMv7 (or newer)", True), + ("armv8a", "ARMv7 (or newer)", True), + ("x86_64", "x86_64", True), + ("riscv64", "riscv64", True), + ("mips", "mips", False), + ], +) +def test_FTL_detect_no_errors(host, arch, detected_string, supported): """ - confirms only aarch64 package is downloaded for FTL engine + confirms only correct package is downloaded for FTL engine """ - # mock uname to return aarch64 platform - mock_command("uname", {"-m": ("aarch64", "0")}, host) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected AArch64 (64 Bit ARM) architecture" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv6_no_errors(host): - """ - confirms only armv6 package is downloaded for FTL engine - """ - # mock uname to return armv6 platform - mock_command("uname", {"-m": ("armv6", "0")}, host) - # mock readelf to respond with armv6l CPU architecture + # mock uname to return passed platform + mock_command("uname", {"-m": (arch, "0")}, host) + # mock readelf to respond with passed CPU architecture mock_command_2( "readelf", { - "-A /bin/sh": ("Tag_CPU_arch: armv6", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv6", "0"), + "-A /bin/sh": ("Tag_CPU_arch: " + arch, "0"), + "-A /usr/bin/sh": ("Tag_CPU_arch: " + arch, "0"), }, host, ) + host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch') detectPlatform = host.run( """ - echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) @@ -282,195 +268,30 @@ def test_FTL_detect_armv6_no_errors(host): FTLdetect "${binary}" "${theRest}" """ ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv6 architecture" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv7l_no_errors(host): - """ - confirms only armv7l package is downloaded for FTL engine - """ - # mock uname to return armv7l platform - mock_command("uname", {"-m": ("armv7l", "0")}, host) - # mock readelf to respond with armv7l CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv7l", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv7l", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv7_no_errors(host): - """ - confirms only armv7 package is downloaded for FTL engine - """ - # mock uname to return armv7 platform - mock_command("uname", {"-m": ("armv7", "0")}, host) - # mock readelf to respond with armv7 CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv7", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv7", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + (" Detected ARMv7 (or newer) architecture") - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_armv8a_no_errors(host): - """ - confirms only armv8a package is downloaded for FTL engine - """ - # mock uname to return armv8a platform - mock_command("uname", {"-m": ("armv8a", "0")}, host) - # mock readelf to respond with armv8a CPU architecture - mock_command_2( - "readelf", - { - "-A /bin/sh": ("Tag_CPU_arch: armv8a", "0"), - "-A /usr/bin/sh": ("Tag_CPU_arch: armv8a", "0"), - }, - host, - ) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected ARMv7 (or newer) architecture (armv8a)" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_x86_64_no_errors(host): - """ - confirms only x86_64 package is downloaded for FTL engine - """ - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = info_box + " FTL Checks..." - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Detected x86_64 architecture" - assert expected_stdout in detectPlatform.stdout - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_detect_unknown_no_errors(host): - """confirms only generic package is downloaded for FTL engine""" - # mock uname to return generic platform - mock_command("uname", {"-m": ("mips", "0")}, host) - detectPlatform = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - funcOutput=$(get_binary_name) - binary="pihole-FTL${funcOutput##*pihole-FTL}" - theRest="${funcOutput%pihole-FTL*}" - FTLdetect "${binary}" "${theRest}" - """ - ) - expected_stdout = "Not able to detect architecture (unknown: mips)" - assert expected_stdout in detectPlatform.stdout - - -def test_FTL_download_aarch64_no_errors(host): - """ - confirms only aarch64 package is downloaded for FTL engine - """ - # mock dialog answers and ensure installer dependencies - mock_command("dialog", {"*": ("", "0")}, host) - host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - install_dependent_packages ${INSTALLER_DEPS[@]} - """ - ) - download_binary = host.run( - """ - echo "development-v6" > /etc/pihole/ftlbranch - source /opt/pihole/basic-install.sh - create_pihole_user - FTLinstall "pihole-FTL-aarch64-linux-gnu" - """ - ) - expected_stdout = tick_box + " Downloading and Installing FTL" - assert expected_stdout in download_binary.stdout - assert "error" not in download_binary.stdout.lower() + if supported: + expected_stdout = info_box + " FTL Checks..." + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Detected " + detected_string + " architecture" + assert expected_stdout in detectPlatform.stdout + expected_stdout = tick_box + " Downloading and Installing FTL" + assert expected_stdout in detectPlatform.stdout + else: + expected_stdout = ( + "Not able to detect architecture (unknown: " + detected_string + ")" + ) + assert expected_stdout in detectPlatform.stdout def test_FTL_development_binary_installed_and_responsive_no_errors(host): """ confirms FTL development binary is copied and functional in installed location """ + host.run('echo "' + FTL_BRANCH + '" > /etc/pihole/ftlbranch') host.run( """ - echo "development-v6" > /etc/pihole/ftlbranch source /opt/pihole/basic-install.sh create_pihole_user funcOutput=$(get_binary_name) - echo "development-v6" > /etc/pihole/ftlbranch binary="pihole-FTL${funcOutput##*pihole-FTL}" theRest="${funcOutput%pihole-FTL*}" FTLdetect "${binary}" "${theRest}" From debab10792c7bee8cd6ba18b6c3798c4f03c8a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 7 Jan 2024 21:37:58 +0100 Subject: [PATCH 295/462] Use 204 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/api.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index fe73a608..18a48ce7 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -113,9 +113,8 @@ DeleteSession() { deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in - "200") printf "%b" "A session that was not created cannot be deleted (e.g., empty API password).\n";; + "204") printf "%b" "Session successfully deleted.\n";; "401") printf "%b" "Logout attempt without a valid session. Unauthorized!\n";; - "410") printf "%b" "Session successfully deleted.\n";; esac; fi From 9eb47319f572674c91ce0a49b5240ddb66aee027 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 13 Jan 2024 10:32:36 +0000 Subject: [PATCH 296/462] Bump tox from 4.11.4 to 4.12.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.4 to 4.12.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.4...4.12.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0873d097..59fd13cc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.11.4 +tox == 4.12.0 From b662fd6f0101193a37b79377b077ffa37c33dfc2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 13 Jan 2024 10:43:13 +0000 Subject: [PATCH 297/462] Bump tox from 4.11.4 to 4.12.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.11.4 to 4.12.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.11.4...4.12.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0873d097..59fd13cc 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.11.4 +tox == 4.12.0 From 47f06dfd71824f199af2a9b757f6ac06f9724ef4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 15 Jan 2024 20:53:34 +0100 Subject: [PATCH 298/462] Do not store remote version in versions file if on custom branch. It's always wrong here Signed-off-by: DL6ER --- advanced/Scripts/updatecheck.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 66f1a7ab..d2a338a7 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -26,7 +26,12 @@ function get_local_hash() { } function get_remote_version() { - curl -s "https://api.github.com/repos/pi-hole/${1}/releases/latest" 2> /dev/null | jq --raw-output .tag_name || return 1 + # if ${2} is = "master" we need to use the "latest" endpoint, otherwise, we simply return null + if [[ "${2}" == "master" ]]; then + curl -s "https://api.github.com/repos/pi-hole/${1}/releases/latest" 2> /dev/null | jq --raw-output .tag_name || return 1 + else + echo "null" + fi } @@ -73,7 +78,7 @@ addOrEditKeyValPair "${VERSION_FILE}" "CORE_BRANCH" "${CORE_BRANCH}" CORE_HASH="$(get_local_hash /etc/.pihole)" addOrEditKeyValPair "${VERSION_FILE}" "CORE_HASH" "${CORE_HASH}" -GITHUB_CORE_VERSION="$(get_remote_version pi-hole)" +GITHUB_CORE_VERSION="$(get_remote_version pi-hole "${CORE_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_CORE_VERSION" "${GITHUB_CORE_VERSION}" GITHUB_CORE_HASH="$(get_remote_hash pi-hole "${CORE_BRANCH}")" @@ -91,7 +96,7 @@ addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" WEB_HASH="$(get_local_hash /var/www/html/admin)" addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" -GITHUB_WEB_VERSION="$(get_remote_version web)" +GITHUB_WEB_VERSION="$(get_remote_version web "${WEB_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_WEB_VERSION" "${GITHUB_WEB_VERSION}" GITHUB_WEB_HASH="$(get_remote_hash web "${WEB_BRANCH}")" @@ -108,7 +113,7 @@ addOrEditKeyValPair "${VERSION_FILE}" "FTL_BRANCH" "${FTL_BRANCH}" FTL_HASH="$(pihole-FTL --hash)" addOrEditKeyValPair "${VERSION_FILE}" "FTL_HASH" "${FTL_HASH}" -GITHUB_FTL_VERSION="$(get_remote_version FTL)" +GITHUB_FTL_VERSION="$(get_remote_version FTL "${FTL_BRANCH}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_FTL_VERSION" "${GITHUB_FTL_VERSION}" GITHUB_FTL_HASH="$(get_remote_hash FTL "${FTL_BRANCH}")" From ba2682c907e446138f142c64b2bef4a617e79692 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jan 2024 10:38:21 +0000 Subject: [PATCH 299/462] Bump tox from 4.12.0 to 4.12.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.12.0 to 4.12.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.12.0...4.12.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 59fd13cc..dea4ace9 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.12.0 +tox == 4.12.1 From 60de50bb73f3df5f78c68726c3cd31ba1d5193ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Jan 2024 10:49:27 +0000 Subject: [PATCH 300/462] Bump tox from 4.12.0 to 4.12.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.12.0 to 4.12.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.12.0...4.12.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 59fd13cc..dea4ace9 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 7.4.4 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.12.0 +tox == 4.12.1 From 935a4ce0b3262fa3b3d468f018771a5001ffe145 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 28 Jan 2024 16:56:57 +0100 Subject: [PATCH 301/462] Also remove Fedora 37 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 3 +-- test/_fedora_37.Dockerfile | 18 ------------------ test/tox.fedora_37.ini | 8 -------- 3 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 test/_fedora_37.Dockerfile delete mode 100644 test/tox.fedora_37.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9d90b2e7..eadfc930 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,8 +64,7 @@ jobs: ubuntu_23, centos_8, centos_9, - fedora_37, - fedora_38, + fedora_38, fedora_39, ] env: diff --git a/test/_fedora_37.Dockerfile b/test/_fedora_37.Dockerfile deleted file mode 100644 index b4f939ba..00000000 --- a/test/_fedora_37.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM fedora:37 -RUN dnf install -y git initscripts - -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole - -RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole -ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR - -RUN true && \ - chmod +x $SCRIPTDIR/* - -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net - -#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini deleted file mode 100644 index 9c8752cc..00000000 --- a/test/tox.fedora_37.ini +++ /dev/null @@ -1,8 +0,0 @@ -[tox] -envlist = py3 - -[testenv] -allowlist_externals = docker -deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From 07b448d784d3f5e11e7a18e32eeab58aa0a5d25a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Jan 2024 17:15:36 +0100 Subject: [PATCH 302/462] Also check for IPv6 address for configured DNS servers Signed-off-by: DL6ER --- automated install/basic-install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f5bf15e6..4bb4106b 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -817,12 +817,12 @@ If you want to specify a port other than 53, separate it with a hash.\ printf -v PIHOLE_DNS_1 "%s" "${piholeDNS%%,*}" printf -v PIHOLE_DNS_2 "%s" "${piholeDNS##*,}" - # If the first DNS value is invalid or empty, this if statement will be true and we will set PIHOLE_DNS_1="Invalid" - if ! valid_ip "${PIHOLE_DNS_1}" || [[ ! "${PIHOLE_DNS_1}" ]]; then + # If the first DNS value is invalid (neither IPv4 nor IPv6) or empty, set PIHOLE_DNS_1="Invalid" + if ! valid_ip "${PIHOLE_DNS_1}" && ! valid_ip6 "${PIHOLE_DNS_1}" || [[ -z "${PIHOLE_DNS_1}" ]]; then PIHOLE_DNS_1=${strInvalid} fi - # If the second DNS value is invalid or empty, this if statement will be true and we will set PIHOLE_DNS_2="Invalid" - if ! valid_ip "${PIHOLE_DNS_2}" && [[ "${PIHOLE_DNS_2}" ]]; then + # If the second DNS value is invalid but not empty, set PIHOLE_DNS_2="Invalid" + if ! valid_ip "${PIHOLE_DNS_2}" && ! valid_ip6 "${PIHOLE_DNS_2}" && [[ -n "${PIHOLE_DNS_2}" ]]; then PIHOLE_DNS_2=${strInvalid} fi # If either of the DNS servers are invalid, From 31a8f150b2e3a5411871f1163c7cc7a433e7ecc5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 30 Jan 2024 19:09:14 +0100 Subject: [PATCH 303/462] Add checking for availability of ftl.pi-hole.net when using FTL from a custom branch. If the server is down (or the user is offline, cannot resolve the domain, etc.), we fail early and hard instead of possibly corrupting the installation Signed-off-by: DL6ER --- advanced/Scripts/piholeCheckout.sh | 22 +++++++++++----- advanced/Scripts/update.sh | 7 ++++- automated install/basic-install.sh | 41 +++++++++++++++++++++--------- 3 files changed, 51 insertions(+), 19 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 39d39b1c..41fd8606 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -164,7 +164,9 @@ checkout() { path="${2}/${binary}" oldbranch="$(pihole-FTL -b)" - if check_download_exists "$path"; then + check_download_exists "$path" + local ret=$? + if [ $ret -eq 0 ]; then echo " ${TICK} Branch ${2} exists" echo "${2}" > /etc/pihole/ftlbranch chmod 644 /etc/pihole/ftlbranch @@ -175,11 +177,19 @@ checkout() { # Update local and remote versions via updatechecker /opt/pihole/updatecheck.sh else - echo " ${CROSS} Requested branch \"${2}\" is not available" - ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') ) - echo -e " ${INFO} Available branches for FTL are:" - for e in "${ftlbranches[@]}"; do echo " - $e"; done - exit 1 + if [[ $ret -eq 1 ]]; then + echo " ${CROSS} Requested branch \"${2}\" is not available" + ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') ) + echo -e " ${INFO} Available branches for FTL are:" + for e in "${ftlbranches[@]}"; do echo " - $e"; done + exit 1 + elif [[ $ret -eq 2 ]]; then + printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" + exit 1 + else + printf " %b Unknown error. Please contact Pi-hole Support\\n" "${CROSS}" + exit 1 + fi fi else diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 9dae66df..8a35ef2e 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -144,7 +144,7 @@ main() { local binary binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL) - if FTLcheckUpdate "${binary}" > /dev/null; then + if FTLcheckUpdate "${binary}"; then FTL_update=true echo -e " ${INFO} FTL:\\t\\t${COL_YELLOW}update available${COL_NC}" else @@ -155,8 +155,13 @@ main() { 2) echo -e " ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Branch is not available.${COL_NC}\\n\\t\\t\\tUse ${COL_LIGHT_GREEN}pihole checkout ftl [branchname]${COL_NC} to switch to a valid branch." ;; + 3) + echo -e " ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Something has gone wrong, cannot reach download server${COL_NC}" + exit 1 + ;; *) echo -e " ${INFO} FTL:\\t\\t${COL_LIGHT_RED}Something has gone wrong, contact support${COL_NC}" + exit 1 esac FTL_update=false fi diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f5bf15e6..af7ca151 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1687,12 +1687,19 @@ update_dialogs() { } check_download_exists() { + # Check if the download exists and we can reach the server status=$(curl --head --silent "https://ftl.pi-hole.net/${1}" | head -n 1) - if grep -q "404" <<< "$status"; then - return 1 - else + + # Check the status code + if grep -q "200" <<< "$status"; then return 0 + elif grep -q "404" <<< "$status"; then + return 1 fi + + # Other error or no status code at all, e.g., no Internet, server not + # available/reachable, ... + return 2 } fully_fetch_repo() { @@ -1957,10 +1964,8 @@ get_binary_name() { } FTLcheckUpdate() { - #In the next section we check to see if FTL is already installed (in case of pihole -r). - #If the installed version matches the latest version, then check the installed sha1sum of the binary vs the remote sha1sum. If they do not match, then download - printf " %b Checking for existing FTL binary...\\n" "${INFO}" - + # In the next section we check to see if FTL is already installed (in case of pihole -r). + # If the installed version matches the latest version, then check the installed sha1sum of the binary vs the remote sha1sum. If they do not match, then download local ftlLoc ftlLoc=$(command -v pihole-FTL 2>/dev/null) @@ -1979,14 +1984,24 @@ FTLcheckUpdate() { local localSha1 if [[ ! "${ftlBranch}" == "master" ]]; then - #Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! + # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! local path path="${ftlBranch}/${binary}" # shellcheck disable=SC1090 - if ! check_download_exists "$path"; then - printf " %b Branch \"%s\" is not available.\\n" "${INFO}" "${ftlBranch}" - printf " %b Use %bpihole checkout ftl [branchname]%b to switch to a valid branch.\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}" - return 2 + check_download_exists "$path" + local ret=$? + if [ $ret -ne 0 ]; then + if [[ $ret -eq 1 ]]; then + printf " %b Branch \"%s\" is not available.\\n" "${INFO}" "${ftlBranch}" + printf " %b Use %bpihole checkout ftl [branchname]%b to switch to a valid branch.\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}" + return 2 + elif [[ $ret -eq 2 ]]; then + printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" + return 3 + else + printf " %b Unknown error. Please contact Pi-hole Support\\n" "${CROSS}" + return 4 + fi fi if [[ ${ftlLoc} ]]; then @@ -2011,12 +2026,14 @@ FTLcheckUpdate() { FTLversion=$(/usr/bin/pihole-FTL tag) local FTLlatesttag + # Get the latest version from the GitHub API if ! FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep --color=never -i Location: | awk -F / '{print $NF}' | tr -d '[:cntrl:]'); then # There was an issue while retrieving the latest version printf " %b Failed to retrieve latest FTL release metadata" "${CROSS}" return 3 fi + # Check if the installed version matches the latest version if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then return 0 else From 6e946f76d6e0166d6f2d0e86ed7ab01686faf6bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 3 Feb 2024 10:21:48 +0000 Subject: [PATCH 304/462] Bump pytest from 7.4.4 to 8.0.0 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.4 to 8.0.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.4...8.0.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index dea4ace9..2b8dfb1b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.4 +pytest == 8.0.0 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.12.1 From f8bfd59f11332b664c954ebfb2ed1586fbd3d070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 7 Nov 2023 22:24:34 +0100 Subject: [PATCH 305/462] Drop Fedora 36 and add Fedora 39 to the test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 2 +- test/{_fedora_36.Dockerfile => _fedora_39.Dockerfile} | 2 +- test/{tox.fedora_36.ini => tox.fedora_39.ini} | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) rename test/{_fedora_36.Dockerfile => _fedora_39.Dockerfile} (97%) rename test/{tox.fedora_36.ini => tox.fedora_39.ini} (81%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 35ed0c30..67377255 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,9 +57,9 @@ jobs: ubuntu_23, centos_8, centos_9, - fedora_36, fedora_37, fedora_38, + fedora_39, ] env: DISTRO: ${{matrix.distro}} diff --git a/test/_fedora_36.Dockerfile b/test/_fedora_39.Dockerfile similarity index 97% rename from test/_fedora_36.Dockerfile rename to test/_fedora_39.Dockerfile index 847767e7..1727a3aa 100644 --- a/test/_fedora_36.Dockerfile +++ b/test/_fedora_39.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:36 +FROM fedora:39 RUN dnf install -y git initscripts ENV GITDIR /etc/.pihole diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_39.ini similarity index 81% rename from test/tox.fedora_36.ini rename to test/tox.fedora_39.ini index 515487ed..7a538371 100644 --- a/test/tox.fedora_36.ini +++ b/test/tox.fedora_39.ini @@ -1,8 +1,8 @@ [tox] envlist = py3 -[testenv:py3] +[testenv] allowlist_externals = docker deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_39.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py From 7b6f0d1596f879b65552c5deddc0dc620b290d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 28 Jan 2024 16:56:57 +0100 Subject: [PATCH 306/462] Also remove Fedora 37 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 3 +-- test/_fedora_37.Dockerfile | 18 ------------------ test/tox.fedora_37.ini | 8 -------- 3 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 test/_fedora_37.Dockerfile delete mode 100644 test/tox.fedora_37.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 67377255..fabf380e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -57,8 +57,7 @@ jobs: ubuntu_23, centos_8, centos_9, - fedora_37, - fedora_38, + fedora_38, fedora_39, ] env: diff --git a/test/_fedora_37.Dockerfile b/test/_fedora_37.Dockerfile deleted file mode 100644 index b4f939ba..00000000 --- a/test/_fedora_37.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM fedora:37 -RUN dnf install -y git initscripts - -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole - -RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole -ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR - -RUN true && \ - chmod +x $SCRIPTDIR/* - -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net - -#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini deleted file mode 100644 index 2a8ef398..00000000 --- a/test/tox.fedora_37.ini +++ /dev/null @@ -1,8 +0,0 @@ -[tox] -envlist = py3 - -[testenv] -allowlist_externals = docker -deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py From f0878c0890926e8f7443c7349581b1b10c8d3e3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 4 Feb 2024 21:39:17 +0000 Subject: [PATCH 307/462] Bump pytest from 7.4.4 to 8.0.0 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.4.4 to 8.0.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.4...8.0.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index dea4ace9..2b8dfb1b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 7.4.4 +pytest == 8.0.0 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.12.1 From b5ab8ac1980442f19d474f630f47da6dd851f151 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 11 Feb 2024 16:54:22 +0100 Subject: [PATCH 308/462] Change UNIQUEness constraint from (address) to (address, type) in the adlist table. This will allow certain adlists to be associated to different groups. A possible scenario is an adlist meant to block a specific service (e.g. Twitter, Youtube, etc.). It can then either be used to ensure these services are really blocked on the devices of group A but will never be blocked on devices of group B. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 097b0a78..42060443 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -27,7 +27,7 @@ CREATE TABLE domainlist CREATE TABLE adlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, - address TEXT UNIQUE NOT NULL, + address TEXT NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), @@ -37,7 +37,8 @@ CREATE TABLE adlist invalid_domains INTEGER NOT NULL DEFAULT 0, status INTEGER NOT NULL DEFAULT 0, abp_entries INTEGER NOT NULL DEFAULT 0, - type INTEGER NOT NULL DEFAULT 0 + type INTEGER NOT NULL DEFAULT 0, + UNIQUE(address, type) ); CREATE TABLE adlist_by_group From bcb712b6e3f408b0080768b9c2c77c2a4c7c6dbd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 11 Feb 2024 21:07:15 +0100 Subject: [PATCH 309/462] Fix accidential double -ni from one of the last PRs Signed-off-by: DL6ER --- gravity.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/gravity.sh b/gravity.sh index 592aa801..4d785d8a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -114,7 +114,7 @@ gravity_swap_databases() { # Update timestamp when the gravity table was last updated successfully update_gravity_timestamp() { - output=$({ printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}"; } 2>&1) + output=$({ printf ".timeout 30000\\nINSERT OR REPLACE INTO info (property,value) values ('updated',cast(strftime('%%s', 'now') as int));" | pihole-FTL sqlite3 -ni "${gravityTEMPfile}"; } 2>&1) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -352,8 +352,8 @@ gravity_DownloadBlocklists() { # Retrieve source URLs from gravity database # We source only enabled adlists, SQLite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)" - mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)" + mapfile -t sources <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2>/dev/null)" + mapfile -t sourceIDs <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT id FROM vw_adlist;" 2>/dev/null)" mapfile -t sourceTypes <<<"$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM vw_adlist;" 2>/dev/null)" # Parse source domains from $sources @@ -667,12 +667,12 @@ gravity_Table_Count() { local table="${1}" local str="${2}" local num - num="$(pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" + num="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM ${table};")" if [[ "${table}" == "gravity" ]]; then local unique - unique="$(pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" + unique="$(pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "SELECT COUNT(*) FROM (SELECT DISTINCT domain FROM ${table});")" echo -e " ${INFO} Number of ${str}: ${num} (${COL_BOLD}${unique} unique domains${COL_NC})" - pihole-FTL sqlite3 -ni -ni "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" + pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('gravity_count',${unique});" else echo -e " ${INFO} Number of ${str}: ${num}" fi From 75fadb9b55fbb1454bbddd60d0ed99924200d2d4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 13 Feb 2024 08:55:26 +0100 Subject: [PATCH 310/462] Adlists need to be grouped by both address and type to differentiate between anti-/gravity lists Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 581d2069..df7db893 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -48,7 +48,7 @@ GenerateOutput() { lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') # construct a new json for the gravity results where each object contains the adlist URL and the related domains - gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address) | map({ address: (.[0].address), domains: [.[] | .domain] })') + gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address,.type) | map({ address: (.[0].address), domains: [.[] | .domain] })') # number of objects in each json num_gravity=$(printf %s "${gravity_data}" | jq length) From 9ff43040ec52d9bc1adefa2f4a69775835dfc67c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 13 Feb 2024 09:58:23 +0100 Subject: [PATCH 311/462] Add list type in pihole -q Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index df7db893..493c75ea 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -41,14 +41,14 @@ Options: GenerateOutput() { local data gravity_data lists_data num_gravity num_lists search_type_str - local gravity_data_csv lists_data_csv line current_domain + local gravity_data_csv lists_data_csv line current_domain url type color data="${1}" # construct a new json for the list results where each object contains the domain and the related type lists_data=$(printf %s "${data}" | jq '.search.domains | [.[] | {domain: .domain, type: .type}]') # construct a new json for the gravity results where each object contains the adlist URL and the related domains - gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address,.type) | map({ address: (.[0].address), domains: [.[] | .domain] })') + gravity_data=$(printf %s "${data}" | jq '.search.gravity | group_by(.address,.type) | map({ address: (.[0].address), type: (.[0].type), domains: [.[] | .domain] })') # number of objects in each json num_gravity=$(printf %s "${gravity_data}" | jq length) @@ -78,15 +78,27 @@ GenerateOutput() { if [ "${num_gravity}" -gt 0 ]; then # Convert the data to a csv, each line is a "URL,domain,domain,...." string # not using jq's @csv here as it quotes each value individually - gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .domains[]] | join(",")') + gravity_data_csv=$(printf %s "${gravity_data}" | jq --raw-output '.[] | [.address, .type, .domains[]] | join(",")') # Generate line-by-line output for each csv line echo "${gravity_data_csv}" | while read -r line; do + # Get first part of the line, the URL + url=${line%%,*} + + # cut off URL, leaving "type,domain,domain,...." + line=${line#*,} + type=${line%%,*} + # type == "block" -> red, type == "allow" -> green + if [ "${type}" = "block" ]; then + color="${COL_RED}" + else + color="${COL_GREEN}" + fi # print adlist URL - printf "%s\n\n" " - ${COL_BLUE}${line%%,*}${COL_NC}" + printf "%s (%s)\n\n" " - ${COL_BLUE}${url}${COL_NC}" "${color}${type}${COL_NC}" - # cut off URL, leaving "domain,domain,...." + # cut off type, leaving "domain,domain,...." line=${line#*,} # print each domain and remove it from the string until nothing is left while [ ${#line} -gt 0 ]; do From 5b75cb1950544b4af0e1e4330fb39f22f4215aca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Feb 2024 10:29:46 +0000 Subject: [PATCH 312/462] Bump tox from 4.12.1 to 4.13.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.12.1 to 4.13.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.12.1...4.13.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 2b8dfb1b..a6826838 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.0.0 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.12.1 +tox == 4.13.0 From b322f1e98bf1ec64b705add4f089cda22ae8e5e1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Feb 2024 10:50:41 +0000 Subject: [PATCH 313/462] Bump tox from 4.12.1 to 4.13.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.12.1 to 4.13.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.12.1...4.13.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 2b8dfb1b..a6826838 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.0.0 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 -tox == 4.12.1 +tox == 4.13.0 From 3ed29f494b93be8b03bc9739ad5f03a7cf578a25 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Feb 2024 12:45:22 +0000 Subject: [PATCH 314/462] Bump pytest from 8.0.0 to 8.0.1 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.0.0 to 8.0.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.0...8.0.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index a6826838..3e341719 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.0.0 +pytest == 8.0.1 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.13.0 From 07e6c0d250febddf706e5b414466b88a6c89b288 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 24 Feb 2024 12:49:07 +0000 Subject: [PATCH 315/462] Bump pytest-testinfra from 10.0.0 to 10.1.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 10.0.0 to 10.1.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/10.0.0...10.1.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 3e341719..2a559b05 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 8.0.1 pytest-xdist == 3.5.0 -pytest-testinfra == 10.0.0 +pytest-testinfra == 10.1.0 tox == 4.13.0 From fd1372df3e72509e16c92bfcb9b4489d9b232fc6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Mar 2024 10:01:31 +0000 Subject: [PATCH 316/462] Bump pytest from 8.0.1 to 8.0.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.0.1 to 8.0.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.1...8.0.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 2a559b05..ffb05813 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.0.1 +pytest == 8.0.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 tox == 4.13.0 From cb3e448b385672f4abb8dbfb78dd0fbb063ac59e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Mar 2024 10:18:47 +0000 Subject: [PATCH 317/462] Bump pytest from 8.0.0 to 8.0.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.0.0 to 8.0.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.0...8.0.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index a6826838..bf9827ad 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.0.0 +pytest == 8.0.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.0.0 tox == 4.13.0 From 0fdd959c7f751d40d9e3fd5947612478f7a5c643 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 2 Mar 2024 11:52:21 +0000 Subject: [PATCH 318/462] Bump pytest-testinfra from 10.0.0 to 10.1.0 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 10.0.0 to 10.1.0. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/10.0.0...10.1.0) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index bf9827ad..ffb05813 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 8.0.2 pytest-xdist == 3.5.0 -pytest-testinfra == 10.0.0 +pytest-testinfra == 10.1.0 tox == 4.13.0 From 9dd138b03348f24a001d60f27e29c8c62af28871 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 4 Mar 2024 19:38:13 +0100 Subject: [PATCH 319/462] Only use local files (file://) when they have explicit permissions a+r Signed-off-by: DL6ER --- gravity.sh | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 62631e3c..86c7fc23 100755 --- a/gravity.sh +++ b/gravity.sh @@ -503,6 +503,7 @@ compareLists() { gravity_DownloadBlocklistFromUrl() { local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext + local file_path permissions ip_addr port blocked=false download=true # Create temp file to store content on disk instead of RAM # We don't use '--suffix' here because not all implementations of mktemp support it, e.g. on Alpine @@ -519,7 +520,6 @@ gravity_DownloadBlocklistFromUrl() { str="Status:" echo -ne " ${INFO} ${str} Pending..." - blocked=false case $BLOCKINGMODE in "IP-NODATA-AAAA"|"IP") # Get IP address of this domain @@ -560,8 +560,36 @@ gravity_DownloadBlocklistFromUrl() { cmd_ext="--resolve $domain:$port:$ip" fi - # shellcheck disable=SC2086 - httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) + # If we are going to "download" a local file, we first check if the target + # file has a+r permission. We explicitly check for all+read because we want + # to make sure that the file is readable by everyone and not just the user + # running the script. + if [[ $url == "file://"* ]]; then + # Get the file path + file_path=$(echo "$url" | cut -d'/' -f3-) + # Check if the file exists + if [[ ! -e $file_path ]]; then + # Output that the file does not exist + echo -e "${OVER} ${CROSS} ${file_path} does not exist" + download=false + else + # Check if the file has a+r permissions + permissions=$(stat -c "%a" "$file_path") + if [[ $permissions == "??4" || $permissions == "??5" || $permissions == "??6" || $permissions == "??7" ]]; then + # Output that we are using the local file + echo -e "${OVER} ${INFO} Using local file ${file_path}" + else + # Output that the file does not have the correct permissions + echo -e "${OVER} ${CROSS} Cannot read file (file needs to have a+r permission)" + download=false + fi + fi + fi + + if [[ "${download}" == true ]]; then + # shellcheck disable=SC2086 + httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) + fi case $url in # Did we "download" a local file? @@ -569,7 +597,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ -s "${listCurlBuffer}" ]]; then echo -e "${OVER} ${TICK} ${str} Retrieval successful"; success=true else - echo -e "${OVER} ${CROSS} ${str} Not found / empty list" + echo -e "${OVER} ${CROSS} ${str} Retrieval failed / empty list" fi;; # Did we "download" a remote file? *) From bfd8b572cbc20a294a2886f1cfb4e2315d10ca98 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Mar 2024 10:07:37 +0000 Subject: [PATCH 320/462] Bump tox from 4.13.0 to 4.14.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.13.0 to 4.14.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.13.0...4.14.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index ffb05813..d4415e2b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.0.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 -tox == 4.13.0 +tox == 4.14.1 From 8042d9e3fd0464964f5f3a783bd1af1db17cc277 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 9 Mar 2024 10:21:53 +0000 Subject: [PATCH 321/462] Bump tox from 4.13.0 to 4.14.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.13.0 to 4.14.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.13.0...4.14.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index ffb05813..d4415e2b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.0.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 -tox == 4.13.0 +tox == 4.14.1 From c02401b81e43febae7365f960c2d530afc714e24 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 9 Mar 2024 11:17:33 +0100 Subject: [PATCH 322/462] Highlight "### CHANGED" strings in the debug log of pihole.toml Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index f5a57278..7c558127 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -865,8 +865,6 @@ make_array_from_file() { local limit=${2} # A local iterator for testing if we are at the limit above local i=0 - # Set the array to be empty so we can start fresh when the function is used - local file_content=() # If the file is a directory if [[ -d "${filename}" ]]; then # do nothing since it cannot be parsed @@ -878,11 +876,14 @@ make_array_from_file() { new_line=$(echo "${line}" | sed -e 's/^\s*#.*$//' -e '/^$/d') # If the line still has content (a non-zero value) if [[ -n "${new_line}" ]]; then - # Put it into the array - file_content+=("${new_line}") - else - # Otherwise, it's a blank line or comment, so do nothing - : + + # If the string contains "### CHANGED", highlight this part in red + if [[ "${new_line}" == *"### CHANGED"* ]]; then + new_line="${new_line//### CHANGED/${COL_RED}### CHANGED${COL_NC}}" + fi + + # Finally, write this line to the log + log_write " ${new_line}" fi # Increment the iterator +1 i=$((i+1)) @@ -894,12 +895,6 @@ make_array_from_file() { break fi done < "${filename}" - # Now the we have made an array of the file's content - for each_line in "${file_content[@]}"; do - # Print each line - # At some point, we may want to check the file line-by-line, so that's the reason for an array - log_write " ${each_line}" - done fi } From 2fd0de4743b134ad1a0be6bea119301d16b98ef5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 10 Mar 2024 08:43:37 +0100 Subject: [PATCH 323/462] Verify that we actually downloaded a valid checksum before comparing it to the local one. This covers situations where downloading the checksum from remote might have failed Signed-off-by: DL6ER --- automated install/basic-install.sh | 38 +++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 24abb7e7..eca7d8b0 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2002,9 +2002,11 @@ FTLcheckUpdate() { local localSha1 if [[ ! "${ftlBranch}" == "master" ]]; then - # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! + # This is not the master branch local path path="${ftlBranch}/${binary}" + + # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! # shellcheck disable=SC1090 check_download_exists "$path" local ret=$? @@ -2023,12 +2025,20 @@ FTLcheckUpdate() { fi if [[ ${ftlLoc} ]]; then - # We already have a pihole-FTL binary downloaded. - # Alt branches don't have a tagged version against them, so just confirm the checksum of the local vs remote to decide whether we download or not + # We already have a pihole-FTL binary installed, check if it's the + # same as the remote one + # Alt branches don't have a tagged version against them, so just + # confirm the checksum of the local vs remote to decide whether we + # download or not remoteSha1=$(curl -sSL --fail "https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "$(command -v pihole-FTL)" | cut -d ' ' -f 1) + localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) - if [[ "${remoteSha1}" != "${localSha1}" ]]; then + # Check we downloaded a valid checksum (no 404 or other error like + # no DNS resolution) + if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then + printf " %b Remote checksum not available, trying to download binary from ftl.pi-hole.net.\\n" "${CROSS}" + return 0 + elif [[ "${remoteSha1}" != "${localSha1}" ]]; then printf " %b Checksums do not match, downloading from ftl.pi-hole.net.\\n" "${INFO}" return 0 else @@ -2039,7 +2049,10 @@ FTLcheckUpdate() { return 0 fi else + # This is the master branch if [[ ${ftlLoc} ]]; then + # We already have a pihole-FTL binary installed, check if it's the + # same as the remote one local FTLversion FTLversion=$(/usr/bin/pihole-FTL tag) local FTLlatesttag @@ -2053,15 +2066,24 @@ FTLcheckUpdate() { # Check if the installed version matches the latest version if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then + # If the installed version does not match the latest version, then download return 0 else + # If the installed version matches the latest version, then + # check the installed sha1sum of the binary vs the remote + # sha1sum. If they do not match, then download printf " %b Latest FTL Binary already installed (%s). Confirming Checksum...\\n" "${INFO}" "${FTLlatesttag}" remoteSha1=$(curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "$(command -v pihole-FTL)" | cut -d ' ' -f 1) + localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) - if [[ "${remoteSha1}" != "${localSha1}" ]]; then - printf " %b Corruption detected...\\n" "${INFO}" + # Check we downloaded a valid checksum (no 404 or other error like + # no DNS resolution) + if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then + printf " %b Remote checksum not available, trying to redownload binary...\\n" "${CROSS}" + return 0 + elif [[ "${remoteSha1}" != "${localSha1}" ]]; then + printf " %b Corruption detected, redownloading binary...\\n" "${CROSS}" return 0 else printf " %b Checksum correct. No need to download!\\n" "${INFO}" From 82a83c497dd772cf6f044ec50b4471af102bff37 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 10 Mar 2024 21:18:13 +0100 Subject: [PATCH 324/462] Reduce code-duplication Signed-off-by: DL6ER --- automated install/basic-install.sh | 59 +++++++++++++----------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index eca7d8b0..d057cb82 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2030,21 +2030,9 @@ FTLcheckUpdate() { # Alt branches don't have a tagged version against them, so just # confirm the checksum of the local vs remote to decide whether we # download or not - remoteSha1=$(curl -sSL --fail "https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) - - # Check we downloaded a valid checksum (no 404 or other error like - # no DNS resolution) - if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then - printf " %b Remote checksum not available, trying to download binary from ftl.pi-hole.net.\\n" "${CROSS}" - return 0 - elif [[ "${remoteSha1}" != "${localSha1}" ]]; then - printf " %b Checksums do not match, downloading from ftl.pi-hole.net.\\n" "${INFO}" - return 0 - else - printf " %b Checksum of installed binary matches remote. No need to download!\\n" "${INFO}" - return 1 - fi + printf " %b FTL binary already installed. Confirming Checksum...\\n" "${INFO}" + checkSumFile="https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1" + # Continue further down... else return 0 fi @@ -2066,34 +2054,39 @@ FTLcheckUpdate() { # Check if the installed version matches the latest version if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then - # If the installed version does not match the latest version, then download + # If the installed version does not match the latest version, + # then download return 0 else # If the installed version matches the latest version, then # check the installed sha1sum of the binary vs the remote # sha1sum. If they do not match, then download - printf " %b Latest FTL Binary already installed (%s). Confirming Checksum...\\n" "${INFO}" "${FTLlatesttag}" - - remoteSha1=$(curl -sSL --fail "https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1" | cut -d ' ' -f 1) - localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) - - # Check we downloaded a valid checksum (no 404 or other error like - # no DNS resolution) - if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then - printf " %b Remote checksum not available, trying to redownload binary...\\n" "${CROSS}" - return 0 - elif [[ "${remoteSha1}" != "${localSha1}" ]]; then - printf " %b Corruption detected, redownloading binary...\\n" "${CROSS}" - return 0 - else - printf " %b Checksum correct. No need to download!\\n" "${INFO}" - return 1 - fi + printf " %b Latest FTL binary already installed (%s). Confirming Checksum...\\n" "${INFO}" "${FTLlatesttag}" + checkSumFile="https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1" + # Continue further down... fi else return 0 fi fi + + # If we reach this point, we need to check the checksum of the local vs + # remote to decide whether we download or not + remoteSha1=$(curl -sSL --fail "${checkSumFile}" | cut -d ' ' -f 1) + localSha1=$(sha1sum "${ftlLoc}" | cut -d ' ' -f 1) + + # Check we downloaded a valid checksum (no 404 or other error like + # no DNS resolution) + if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then + printf " %b Remote checksum not available, trying to redownload binary...\\n" "${CROSS}" + return 0 + elif [[ "${remoteSha1}" != "${localSha1}" ]]; then + printf " %b Corruption detected, redownloading binary...\\n" "${CROSS}" + return 0 + fi + + printf " %b Checksum correct. No need to download!\\n" "${INFO}" + return 1 } # Detect suitable FTL binary platform From eb23fbf8796e63deda951b52af55d3969cc4bf44 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Mar 2024 10:08:27 +0000 Subject: [PATCH 325/462] Bump actions/checkout from 4.1.1 to 4.1.2 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.1 to 4.1.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.1...v4.1.2) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index fc821194..9cfd8a61 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 095d7358..c6a581ff 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index e52d4ae9..9b35a974 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fabf380e..d2282d2d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Check scripts in repository are executable run: | @@ -64,7 +64,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Set up Python 3.10 uses: actions/setup-python@v5.0.0 From 58a1e8582611f70249ef5592286b5423c5106436 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Mar 2024 10:21:49 +0000 Subject: [PATCH 326/462] Bump actions/checkout from 4.1.1 to 4.1.2 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.1 to 4.1.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.1...v4.1.2) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index fc821194..9cfd8a61 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 095d7358..c6a581ff 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index e52d4ae9..9b35a974 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a2b50dc..bf52d46f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Check scripts in repository are executable run: | @@ -71,7 +71,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.1 + uses: actions/checkout@v4.1.2 - name: Set up Python 3.10 uses: actions/setup-python@v5.0.0 From 08dd321f735867ec22e788fdb7b1829d87422905 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 16 Mar 2024 10:44:52 +0000 Subject: [PATCH 327/462] Bump pytest from 8.0.2 to 8.1.1 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.0.2 to 8.1.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.2...8.1.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index d4415e2b..c992752e 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.0.2 +pytest == 8.1.1 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 tox == 4.14.1 From 7aab7d984939d098d61fb2735009c6ee3386a1cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Mar 2024 10:34:48 +0000 Subject: [PATCH 328/462] Bump tox from 4.14.1 to 4.14.2 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.14.1 to 4.14.2. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.14.1...4.14.2) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index d4415e2b..f8d5d94c 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.0.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 -tox == 4.14.1 +tox == 4.14.2 From 472602ffb9d84bcc4b26a088d880eda04c2d60de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 23 Mar 2024 10:52:31 +0000 Subject: [PATCH 329/462] Bump tox from 4.14.1 to 4.14.2 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.14.1 to 4.14.2. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.14.1...4.14.2) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index d4415e2b..f8d5d94c 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.0.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 -tox == 4.14.1 +tox == 4.14.2 From eb7daf4d2c5ca9f77b70f11c80a2b8b03ca65af5 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 27 Mar 2024 19:12:59 +0000 Subject: [PATCH 330/462] Fix file permission check in gravity.sh. Remove quotes that were added after complaints from shellcheck, this stopped the comparisson from working Signed-off-by: Adam Warner --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 86c7fc23..ccb082e2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -575,7 +575,7 @@ gravity_DownloadBlocklistFromUrl() { else # Check if the file has a+r permissions permissions=$(stat -c "%a" "$file_path") - if [[ $permissions == "??4" || $permissions == "??5" || $permissions == "??6" || $permissions == "??7" ]]; then + if [[ $permissions == *4 || $permissions == *5 || $permissions == *6 || $permissions == *7 ]]; then # Output that we are using the local file echo -e "${OVER} ${INFO} Using local file ${file_path}" else From 9f9e5dffc259d7509bf5f1fdfb59be9c8da0cc57 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Mar 2024 20:27:30 +0000 Subject: [PATCH 331/462] Bump pytest from 8.0.2 to 8.1.1 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.0.2 to 8.1.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.0.2...8.1.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index f8d5d94c..9426dd2c 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.0.2 +pytest == 8.1.1 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 tox == 4.14.2 From d80fcf2e62e4610dea0249c3fe3b521f4da820fb Mon Sep 17 00:00:00 2001 From: Orazio <22700499+orazioedoardo@users.noreply.github.com> Date: Wed, 27 Mar 2024 22:10:12 +0100 Subject: [PATCH 332/462] More checks when downloading from file:// scheme Signed-off-by: Orazio <22700499+orazioedoardo@users.noreply.github.com> --- gravity.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index ccb082e2..636cde0d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -567,14 +567,14 @@ gravity_DownloadBlocklistFromUrl() { if [[ $url == "file://"* ]]; then # Get the file path file_path=$(echo "$url" | cut -d'/' -f3-) - # Check if the file exists - if [[ ! -e $file_path ]]; then + # Check if the file exists and is a regular file (i.e. not a socket, fifo, tty, block). Might still be a symlink. + if [[ ! -f $file_path ]]; then # Output that the file does not exist echo -e "${OVER} ${CROSS} ${file_path} does not exist" download=false else - # Check if the file has a+r permissions - permissions=$(stat -c "%a" "$file_path") + # Check if the file or a file referenced by the symlink has a+r permissions + permissions=$(stat -L -c "%a" "$file_path") if [[ $permissions == *4 || $permissions == *5 || $permissions == *6 || $permissions == *7 ]]; then # Output that we are using the local file echo -e "${OVER} ${INFO} Using local file ${file_path}" From 398f90f7f436c6773f003692857e62aabf8a6d6d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 28 Mar 2024 09:12:51 +0100 Subject: [PATCH 333/462] Non-matching checksums are not always corruption. Actually, they will instead be caused by binaries updated on the remote branch. This is most seen with frequently channging branchs such as development-v6 at this time. Signed-off-by: DL6ER --- automated install/basic-install.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index d057cb82..41cc2161 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2030,7 +2030,7 @@ FTLcheckUpdate() { # Alt branches don't have a tagged version against them, so just # confirm the checksum of the local vs remote to decide whether we # download or not - printf " %b FTL binary already installed. Confirming Checksum...\\n" "${INFO}" + printf " %b FTL binary already installed, verifying integrity...\\n" "${INFO}" checkSumFile="https://ftl.pi-hole.net/${ftlBranch}/${binary}.sha1" # Continue further down... else @@ -2061,7 +2061,7 @@ FTLcheckUpdate() { # If the installed version matches the latest version, then # check the installed sha1sum of the binary vs the remote # sha1sum. If they do not match, then download - printf " %b Latest FTL binary already installed (%s). Confirming Checksum...\\n" "${INFO}" "${FTLlatesttag}" + printf " %b Latest FTL binary already installed (%s), verifying integrity...\\n" "${INFO}" "${FTLlatesttag}" checkSumFile="https://github.com/pi-hole/FTL/releases/download/${FTLversion%$'\r'}/${binary}.sha1" # Continue further down... fi @@ -2078,14 +2078,14 @@ FTLcheckUpdate() { # Check we downloaded a valid checksum (no 404 or other error like # no DNS resolution) if [[ ! "${remoteSha1}" =~ ^[a-f0-9]{40}$ ]]; then - printf " %b Remote checksum not available, trying to redownload binary...\\n" "${CROSS}" + printf " %b Remote checksum not available, trying to redownload...\\n" "${CROSS}" return 0 elif [[ "${remoteSha1}" != "${localSha1}" ]]; then - printf " %b Corruption detected, redownloading binary...\\n" "${CROSS}" + printf " %b Remote binary is different, downloading...\\n" "${CROSS}" return 0 fi - printf " %b Checksum correct. No need to download!\\n" "${INFO}" + printf " %b Local binary up-to-date. No need to download!\\n" "${INFO}" return 1 } From b595b3b5f495223ec04d8fa159c5fcb3ded8b44d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 28 Mar 2024 15:41:45 +0100 Subject: [PATCH 334/462] Port most recent changes of PRs #5615, #5618 amd #5620 to v6 Signed-off-by: DL6ER --- gravity.sh | 39 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index d49af29d..bc0ff23d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -488,6 +488,7 @@ compareLists() { gravity_DownloadBlocklistFromUrl() { local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" domain="${7}" local heisenbergCompensator="" listCurlBuffer str httpCode success="" ip cmd_ext + local file_path permissions ip_addr port blocked=false download=true # Create temp file to store content on disk instead of RAM # We don't use '--suffix' here because not all implementations of mktemp support it, e.g. on Alpine @@ -591,8 +592,40 @@ gravity_DownloadBlocklistFromUrl() { fi fi - # shellcheck disable=SC2086 - httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null) + # If we are going to "download" a local file, we first check if the target + # file has a+r permission. We explicitly check for all+read because we want + # to make sure that the file is readable by everyone and not just the user + # running the script. + if [[ $url == "file://"* ]]; then + # Get the file path + file_path=$(echo "$url" | cut -d'/' -f3-) + # Check if the file exists and is a regular file (or a symlink to one) + if [[ ! -e $file_path ]]; then + # Output that the file does not exist + echo -e "${OVER} ${CROSS} ${file_path} does not exist" + download=false + elif [[ ! -f $file_path ]]; then + # Output that the file is not a regular file + echo -e "${OVER} ${CROSS} ${file_path} is not a regular file" + download=false + else + # Check if the file has a+r permissions + permissions=$(stat -Lc "%a" "$file_path") + if [[ $permissions == *4 || $permissions == *5 || $permissions == *6 || $permissions == *7 ]]; then + # Output that we are using the local file + echo -e "${OVER} ${INFO} Using local file ${file_path}" + else + # Output that the file does not have the correct permissions + echo -e "${OVER} ${CROSS} Cannot read file (file needs to have o+r permission)" + download=false + fi + fi + fi + + if [[ "${download}" == true ]]; then + # shellcheck disable=SC2086 + httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null) + fi case $url in # Did we "download" a local file? @@ -601,7 +634,7 @@ gravity_DownloadBlocklistFromUrl() { echo -e "${OVER} ${TICK} ${str} Retrieval successful" success=true else - echo -e "${OVER} ${CROSS} ${str} Not found / empty list" + echo -e "${OVER} ${CROSS} ${str} Retrieval failed / empty list" fi ;; # Did we "download" a remote file? From 67ddb64bcca77f942158e26e14816cd1bb88600b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 30 Mar 2024 10:10:21 +0000 Subject: [PATCH 335/462] Bump eps1lon/actions-label-merge-conflict from 2.1.0 to 3.0.0 Bumps [eps1lon/actions-label-merge-conflict](https://github.com/eps1lon/actions-label-merge-conflict) from 2.1.0 to 3.0.0. - [Release notes](https://github.com/eps1lon/actions-label-merge-conflict/releases) - [Changelog](https://github.com/eps1lon/actions-label-merge-conflict/blob/main/CHANGELOG.md) - [Commits](https://github.com/eps1lon/actions-label-merge-conflict/compare/v2.1.0...v3.0.0) --- updated-dependencies: - dependency-name: eps1lon/actions-label-merge-conflict dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/merge-conflict.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index d86e9cd1..d9ceacca 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v2.1.0 + uses: eps1lon/actions-label-merge-conflict@v3.0.0 with: dirtyLabel: "PR: Merge Conflict" repoToken: "${{ secrets.GITHUB_TOKEN }}" From 81ad1a76e6ddeb2ced0768bd3187b412f9705ea7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 30 Mar 2024 10:50:22 +0000 Subject: [PATCH 336/462] Bump eps1lon/actions-label-merge-conflict from 2.1.0 to 3.0.0 Bumps [eps1lon/actions-label-merge-conflict](https://github.com/eps1lon/actions-label-merge-conflict) from 2.1.0 to 3.0.0. - [Release notes](https://github.com/eps1lon/actions-label-merge-conflict/releases) - [Changelog](https://github.com/eps1lon/actions-label-merge-conflict/blob/main/CHANGELOG.md) - [Commits](https://github.com/eps1lon/actions-label-merge-conflict/compare/v2.1.0...v3.0.0) --- updated-dependencies: - dependency-name: eps1lon/actions-label-merge-conflict dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/merge-conflict.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index d86e9cd1..d9ceacca 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v2.1.0 + uses: eps1lon/actions-label-merge-conflict@v3.0.0 with: dirtyLabel: "PR: Merge Conflict" repoToken: "${{ secrets.GITHUB_TOKEN }}" From 91e0d668e0a552ca08e14de67937bd0a0986e9d7 Mon Sep 17 00:00:00 2001 From: Dominik Date: Sat, 30 Mar 2024 20:03:15 +0100 Subject: [PATCH 337/462] Code style: Use explicit form of CLI arguments Co-authored-by: Dan Schaper Signed-off-by: Dominik --- gravity.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index bc0ff23d..e37c8104 100755 --- a/gravity.sh +++ b/gravity.sh @@ -598,7 +598,7 @@ gravity_DownloadBlocklistFromUrl() { # running the script. if [[ $url == "file://"* ]]; then # Get the file path - file_path=$(echo "$url" | cut -d'/' -f3-) + file_path=$(echo "$url" | cut --delimiter='/' --fields=3-) # Check if the file exists and is a regular file (or a symlink to one) if [[ ! -e $file_path ]]; then # Output that the file does not exist @@ -610,7 +610,7 @@ gravity_DownloadBlocklistFromUrl() { download=false else # Check if the file has a+r permissions - permissions=$(stat -Lc "%a" "$file_path") + permissions=$(stat --dereference --format="%a" "$file_path") if [[ $permissions == *4 || $permissions == *5 || $permissions == *6 || $permissions == *7 ]]; then # Output that we are using the local file echo -e "${OVER} ${INFO} Using local file ${file_path}" @@ -624,7 +624,7 @@ gravity_DownloadBlocklistFromUrl() { if [[ "${download}" == true ]]; then # shellcheck disable=SC2086 - httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null) + httpCode=$(curl --connect-timeout ${curl_connect_timeout} --silent --location ${compression} ${cmd_ext} ${heisenbergCompensator} --write-out "%{http_code}" "${url}" --output "${listCurlBuffer}" 2>/dev/null) fi case $url in From f01362caf15141b95e85a287dff58b414143a7cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Apr 2024 13:17:20 +0000 Subject: [PATCH 338/462] Bump actions/setup-python from 5.0.0 to 5.1.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.0.0 to 5.1.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.0.0...v5.1.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf52d46f..52acc09a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,7 +74,7 @@ jobs: uses: actions/checkout@v4.1.2 - name: Set up Python 3.10 - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@v5.1.0 with: python-version: "3.10" From 44703bbbc6f3f5b35abb93cba03de11e4bd13af2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Apr 2024 10:05:32 +0000 Subject: [PATCH 339/462] Bump actions/checkout from 4.1.2 to 4.1.3 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.2 to 4.1.3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.2...v4.1.3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9cfd8a61..ee0b7dae 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index c6a581ff..bbd275b8 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 9b35a974..c04b24be 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d2282d2d..61abddd8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 - name: Check scripts in repository are executable run: | @@ -64,7 +64,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 - name: Set up Python 3.10 uses: actions/setup-python@v5.0.0 From 5d77c2b362e5748ef87accb6c63c07bff6edd81d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 20 Apr 2024 10:06:19 +0000 Subject: [PATCH 340/462] Bump actions/checkout from 4.1.2 to 4.1.3 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.2 to 4.1.3. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.2...v4.1.3) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9cfd8a61..ee0b7dae 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index c6a581ff..bbd275b8 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 9b35a974..c04b24be 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 52acc09a..b0454998 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 - name: Check scripts in repository are executable run: | @@ -71,7 +71,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.2 + uses: actions/checkout@v4.1.3 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From 091a6891eb12ce6e69cc06e34dcd1706e3d5722e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 21 Apr 2024 16:24:45 +0000 Subject: [PATCH 341/462] Bump actions/setup-python from 5.0.0 to 5.1.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.0.0 to 5.1.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.0.0...v5.1.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 61abddd8..9468cc71 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,7 +67,7 @@ jobs: uses: actions/checkout@v4.1.3 - name: Set up Python 3.10 - uses: actions/setup-python@v5.0.0 + uses: actions/setup-python@v5.1.0 with: python-version: "3.10" From 91d95e2c8ad2d64c5289b85eccd79ea128391b3f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 10:21:11 +0000 Subject: [PATCH 342/462] Bump actions/checkout from 4.1.3 to 4.1.4 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.3 to 4.1.4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.3...v4.1.4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ee0b7dae..51920384 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index bbd275b8..51289101 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index c04b24be..8e024b10 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b0454998..5eda11ce 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 - name: Check scripts in repository are executable run: | @@ -71,7 +71,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From 09ce0003405945b6e63744a14f9023d5acc1e7d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 10:28:36 +0000 Subject: [PATCH 343/462] Bump tox from 4.14.2 to 4.15.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.14.2 to 4.15.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.14.2...4.15.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 9426dd2c..0269d946 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.1.1 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 -tox == 4.14.2 +tox == 4.15.0 From 3cebb3f060a2bd90a1f259265b080c840b97f8a5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 10:46:03 +0000 Subject: [PATCH 344/462] Bump actions/checkout from 4.1.3 to 4.1.4 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.3 to 4.1.4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.3...v4.1.4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ee0b7dae..51920384 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index bbd275b8..51289101 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index c04b24be..8e024b10 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9468cc71..47e2394a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 - name: Check scripts in repository are executable run: | @@ -64,7 +64,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.3 + uses: actions/checkout@v4.1.4 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From b5a7567f3beaeeafb4cc9774cd920252f57f8abd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 10:53:24 +0000 Subject: [PATCH 345/462] Bump pytest from 8.1.1 to 8.1.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.1.1 to 8.1.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.1.1...8.1.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 9426dd2c..4b38dccd 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.1.1 +pytest == 8.1.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 tox == 4.14.2 From b33bbd7e288bac5ab933b310e64f1def52abf7b9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 10:53:28 +0000 Subject: [PATCH 346/462] Bump tox from 4.14.2 to 4.15.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.14.2 to 4.15.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.14.2...4.15.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 9426dd2c..0269d946 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.1.1 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 -tox == 4.14.2 +tox == 4.15.0 From 78c755b53849ce1a7b21a6935b9bfce704e83bab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 11:03:12 +0000 Subject: [PATCH 347/462] Bump pytest from 8.1.1 to 8.1.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.1.1 to 8.1.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.1.1...8.1.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 0269d946..1b27d448 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.1.1 +pytest == 8.1.2 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 tox == 4.15.0 From bf0c3c0e0fa184a4ae171c03be2be451f0413d35 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 10:04:20 +0000 Subject: [PATCH 348/462] Bump pytest from 8.1.2 to 8.2.0 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.1.2 to 8.2.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.1.2...8.2.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 1b27d448..721029ed 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.1.2 +pytest == 8.2.0 pytest-xdist == 3.5.0 pytest-testinfra == 10.1.0 tox == 4.15.0 From 29d4ed113475a367b149b900d8c6d315501ee9e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 10:15:49 +0000 Subject: [PATCH 349/462] Bump pytest-xdist from 3.5.0 to 3.6.1 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.5.0 to 3.6.1. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.5.0...v3.6.1) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 1b27d448..8313c8d3 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 8.1.2 -pytest-xdist == 3.5.0 +pytest-xdist == 3.6.1 pytest-testinfra == 10.1.0 tox == 4.15.0 From 24ade13c0245e769c7ddb7e909a788044c9bf409 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 10:52:00 +0000 Subject: [PATCH 350/462] Bump pytest-xdist from 3.5.0 to 3.6.1 in /test Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.5.0 to 3.6.1. - [Release notes](https://github.com/pytest-dev/pytest-xdist/releases) - [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.5.0...v3.6.1) --- updated-dependencies: - dependency-name: pytest-xdist dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 721029ed..76af80a3 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 8.2.0 -pytest-xdist == 3.5.0 +pytest-xdist == 3.6.1 pytest-testinfra == 10.1.0 tox == 4.15.0 From 9f7cd050e93267ff1d2e5d558e0860b6f5ad9e50 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 4 May 2024 10:56:23 +0000 Subject: [PATCH 351/462] Bump pytest from 8.1.2 to 8.2.0 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.1.2 to 8.2.0. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.1.2...8.2.0) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 8313c8d3..76af80a3 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.1.2 +pytest == 8.2.0 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.0 tox == 4.15.0 From cd30772c43a86c043ed3a1f5c862e1efd6c28aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 11 May 2024 14:21:51 +0200 Subject: [PATCH 352/462] Add Ubuntu 24.04 and Fedora 40, remove Fedora 38 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 3 ++- ...ora_38.Dockerfile => _fedora_40.Dockerfile} | 2 +- test/_ubuntu_24.Dockerfile | 18 ++++++++++++++++++ test/{tox.fedora_38.ini => tox.fedora_40.ini} | 2 +- test/tox.ubuntu_24.ini | 8 ++++++++ 5 files changed, 30 insertions(+), 3 deletions(-) rename test/{_fedora_38.Dockerfile => _fedora_40.Dockerfile} (97%) create mode 100644 test/_ubuntu_24.Dockerfile rename test/{tox.fedora_38.ini => tox.fedora_40.ini} (84%) create mode 100644 test/tox.ubuntu_24.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5eda11ce..71bd04bd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,10 +62,11 @@ jobs: ubuntu_20, ubuntu_22, ubuntu_23, + ubuntu_24, centos_8, centos_9, - fedora_38, fedora_39, + fedora_40, ] env: DISTRO: ${{matrix.distro}} diff --git a/test/_fedora_38.Dockerfile b/test/_fedora_40.Dockerfile similarity index 97% rename from test/_fedora_38.Dockerfile rename to test/_fedora_40.Dockerfile index 76f69771..6d00072f 100644 --- a/test/_fedora_38.Dockerfile +++ b/test/_fedora_40.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:38 +FROM fedora:40 RUN dnf install -y git initscripts ENV GITDIR /etc/.pihole diff --git a/test/_ubuntu_24.Dockerfile b/test/_ubuntu_24.Dockerfile new file mode 100644 index 00000000..2b048361 --- /dev/null +++ b/test/_ubuntu_24.Dockerfile @@ -0,0 +1,18 @@ +FROM buildpack-deps:24.04-scm + +ENV GITDIR /etc/.pihole +ENV SCRIPTDIR /opt/pihole + +RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole +ADD . $GITDIR +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV DEBIAN_FRONTEND=noninteractive + +RUN true && \ + chmod +x $SCRIPTDIR/* + +ENV SKIP_INSTALL true +ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net + +#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.fedora_38.ini b/test/tox.fedora_40.ini similarity index 84% rename from test/tox.fedora_38.ini rename to test/tox.fedora_40.ini index d596092c..149630d7 100644 --- a/test/tox.fedora_38.ini +++ b/test/tox.fedora_40.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv] allowlist_externals = docker deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_38.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_40.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.ubuntu_24.ini b/test/tox.ubuntu_24.ini new file mode 100644 index 00000000..dbd278d6 --- /dev/null +++ b/test/tox.ubuntu_24.ini @@ -0,0 +1,8 @@ +[tox] +envlist = py3 + +[testenv:py3] +allowlist_externals = docker +deps = -rrequirements.txt +commands = docker buildx build --load --progress plain -f _ubuntu_24.Dockerfile -t pytest_pihole:test_container ../ + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py From f4c9278b86608f9d10496d9c6f647ed3e7145a4a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 May 2024 15:45:15 +0000 Subject: [PATCH 353/462] Bump eps1lon/actions-label-merge-conflict from 3.0.0 to 3.0.1 Bumps [eps1lon/actions-label-merge-conflict](https://github.com/eps1lon/actions-label-merge-conflict) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/eps1lon/actions-label-merge-conflict/releases) - [Changelog](https://github.com/eps1lon/actions-label-merge-conflict/blob/main/CHANGELOG.md) - [Commits](https://github.com/eps1lon/actions-label-merge-conflict/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: eps1lon/actions-label-merge-conflict dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/merge-conflict.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index d9ceacca..491ede68 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v3.0.0 + uses: eps1lon/actions-label-merge-conflict@v3.0.1 with: dirtyLabel: "PR: Merge Conflict" repoToken: "${{ secrets.GITHUB_TOKEN }}" From 3ad90c10066852fccd7fdc1535f2b4c39f7b5eee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 May 2024 15:47:44 +0000 Subject: [PATCH 354/462] Bump actions/checkout from 4.1.4 to 4.1.5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.4 to 4.1.5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.4...v4.1.5) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 51920384..910c0b24 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 51289101..ab647ea2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 8e024b10..f6361037 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 71bd04bd..fec902df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From c0cc9bac356f7272a00f496b5e54cc617bd3cfe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 11 May 2024 14:21:51 +0200 Subject: [PATCH 355/462] Add Ubuntu 24.04 and Fedora 40, remove Fedora 38 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 3 ++- ...ora_38.Dockerfile => _fedora_40.Dockerfile} | 2 +- test/_ubuntu_24.Dockerfile | 18 ++++++++++++++++++ test/{tox.fedora_38.ini => tox.fedora_40.ini} | 2 +- test/tox.ubuntu_24.ini | 8 ++++++++ 5 files changed, 30 insertions(+), 3 deletions(-) rename test/{_fedora_38.Dockerfile => _fedora_40.Dockerfile} (97%) create mode 100644 test/_ubuntu_24.Dockerfile rename test/{tox.fedora_38.ini => tox.fedora_40.ini} (85%) create mode 100644 test/tox.ubuntu_24.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 47e2394a..30caf08c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -55,10 +55,11 @@ jobs: ubuntu_20, ubuntu_22, ubuntu_23, + ubuntu_24, centos_8, centos_9, - fedora_38, fedora_39, + fedora_40, ] env: DISTRO: ${{matrix.distro}} diff --git a/test/_fedora_38.Dockerfile b/test/_fedora_40.Dockerfile similarity index 97% rename from test/_fedora_38.Dockerfile rename to test/_fedora_40.Dockerfile index 76f69771..6d00072f 100644 --- a/test/_fedora_38.Dockerfile +++ b/test/_fedora_40.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:38 +FROM fedora:40 RUN dnf install -y git initscripts ENV GITDIR /etc/.pihole diff --git a/test/_ubuntu_24.Dockerfile b/test/_ubuntu_24.Dockerfile new file mode 100644 index 00000000..2b048361 --- /dev/null +++ b/test/_ubuntu_24.Dockerfile @@ -0,0 +1,18 @@ +FROM buildpack-deps:24.04-scm + +ENV GITDIR /etc/.pihole +ENV SCRIPTDIR /opt/pihole + +RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole +ADD . $GITDIR +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ +ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV DEBIAN_FRONTEND=noninteractive + +RUN true && \ + chmod +x $SCRIPTDIR/* + +ENV SKIP_INSTALL true +ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net + +#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.fedora_38.ini b/test/tox.fedora_40.ini similarity index 85% rename from test/tox.fedora_38.ini rename to test/tox.fedora_40.ini index 0aa7612e..cfab6fb0 100644 --- a/test/tox.fedora_38.ini +++ b/test/tox.fedora_40.ini @@ -4,5 +4,5 @@ envlist = py3 [testenv] allowlist_externals = docker deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _fedora_38.Dockerfile -t pytest_pihole:test_container ../ +commands = docker buildx build --load --progress plain -f _fedora_40.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py diff --git a/test/tox.ubuntu_24.ini b/test/tox.ubuntu_24.ini new file mode 100644 index 00000000..dbd278d6 --- /dev/null +++ b/test/tox.ubuntu_24.ini @@ -0,0 +1,8 @@ +[tox] +envlist = py3 + +[testenv:py3] +allowlist_externals = docker +deps = -rrequirements.txt +commands = docker buildx build --load --progress plain -f _ubuntu_24.Dockerfile -t pytest_pihole:test_container ../ + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py From 7426076297d66fcd1ed940c12010a7a657585242 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 May 2024 20:32:20 +0000 Subject: [PATCH 356/462] Bump eps1lon/actions-label-merge-conflict from 3.0.0 to 3.0.1 Bumps [eps1lon/actions-label-merge-conflict](https://github.com/eps1lon/actions-label-merge-conflict) from 3.0.0 to 3.0.1. - [Release notes](https://github.com/eps1lon/actions-label-merge-conflict/releases) - [Changelog](https://github.com/eps1lon/actions-label-merge-conflict/blob/main/CHANGELOG.md) - [Commits](https://github.com/eps1lon/actions-label-merge-conflict/compare/v3.0.0...v3.0.1) --- updated-dependencies: - dependency-name: eps1lon/actions-label-merge-conflict dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/merge-conflict.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index d9ceacca..491ede68 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v3.0.0 + uses: eps1lon/actions-label-merge-conflict@v3.0.1 with: dirtyLabel: "PR: Merge Conflict" repoToken: "${{ secrets.GITHUB_TOKEN }}" From f478913deef07dbe2381cf76498705c5feef8207 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 11 May 2024 20:36:33 +0000 Subject: [PATCH 357/462] Bump actions/checkout from 4.1.4 to 4.1.5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.4 to 4.1.5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.4...v4.1.5) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 51920384..910c0b24 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 51289101..ab647ea2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index 8e024b10..f6361037 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 30caf08c..e898b406 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Check scripts in repository are executable run: | @@ -65,7 +65,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.4 + uses: actions/checkout@v4.1.5 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From 443c5e8243dd9a51e7cbe5e332416da724270a22 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 May 2024 10:31:36 +0000 Subject: [PATCH 358/462] Bump actions/checkout from 4.1.5 to 4.1.6 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.5 to 4.1.6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.5...v4.1.6) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 910c0b24..dd97f2ee 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index ab647ea2..3e9ccbc3 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index f6361037..d215e80f 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e898b406..92f95320 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Check scripts in repository are executable run: | @@ -65,7 +65,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From 5270336679030dce6f4c3d01201014e178113136 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 May 2024 10:56:18 +0000 Subject: [PATCH 359/462] Bump actions/checkout from 4.1.5 to 4.1.6 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.5 to 4.1.6. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.5...v4.1.6) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 910c0b24..dd97f2ee 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index ab647ea2..3e9ccbc3 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index f6361037..d215e80f 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fec902df..568c4669 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Check scripts in repository are executable run: | @@ -72,7 +72,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.5 + uses: actions/checkout@v4.1.6 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From 01e0c6095926746a3ce787a513d952c5376b05af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 May 2024 10:38:28 +0000 Subject: [PATCH 360/462] Bump pytest from 8.2.0 to 8.2.1 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.2.0 to 8.2.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.2.0...8.2.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 76af80a3..3ece8551 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.2.0 +pytest == 8.2.1 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.0 tox == 4.15.0 From d30a7612abe969da015ad93b651cfff3e6e08c88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 25 May 2024 11:02:34 +0000 Subject: [PATCH 361/462] Bump pytest from 8.2.0 to 8.2.1 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.2.0 to 8.2.1. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.2.0...8.2.1) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 76af80a3..3ece8551 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.2.0 +pytest == 8.2.1 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.0 tox == 4.15.0 From d15beff4ba2d7b65724b07be91ff8f18924a55a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 25 May 2024 21:49:24 +0200 Subject: [PATCH 362/462] Fix codespell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 4c69788f..daa35a70 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1442,7 +1442,7 @@ installConfigs() { # Get the version number of lighttpd version=$(dpkg-query -f='${Version}\n' --show lighttpd) - # Test if that version is greater than or euqal to 1.4.56 + # Test if that version is greater than or equal to 1.4.56 if dpkg --compare-versions "$version" "ge" "1.4.56"; then # If it is, then we don't need to disable the modules # (server.modules duplication is ignored in lighttpd 1.4.56+) From 7b19b650d48854d0c96bda78f58d76c90d25b2e1 Mon Sep 17 00:00:00 2001 From: Jack'lul Date: Sun, 26 May 2024 08:38:03 +0200 Subject: [PATCH 363/462] Fix version check for release Docker images Signed-off-by: Jack'lul --- advanced/Scripts/updatecheck.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index eda6c403..b325ee9c 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -54,7 +54,8 @@ chmod 644 "${VERSION_FILE}" # if /pihole.docker.tag file exists, we will use it's value later in this script DOCKER_TAG=$(cat /pihole.docker.tag 2>/dev/null) -regex='^([0-9]+\.){1,2}(\*|[0-9]+)(-.*)?$|(^nightly$)|(^dev.*$)' +release_regex='^([0-9]+\.){1,2}(\*|[0-9]+)(-.*)?$' +regex=$release_regex'|(^nightly$)|(^dev.*$)' if [[ ! "${DOCKER_TAG}" =~ $regex ]]; then # DOCKER_TAG does not match the pattern (see https://regex101.com/r/RsENuz/1), so unset it. unset DOCKER_TAG @@ -121,6 +122,12 @@ addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_FTL_HASH" "${GITHUB_FTL_HASH}" if [[ "${DOCKER_TAG}" ]]; then addOrEditKeyValPair "${VERSION_FILE}" "DOCKER_VERSION" "${DOCKER_TAG}" - GITHUB_DOCKER_VERSION="$(get_remote_version docker-pi-hole)" + # Remote version check only if the tag is a valid release version + docker_branch="" + if [[ "${DOCKER_TAG}" =~ $release_regex ]]; then + docker_branch="master" + fi + + GITHUB_DOCKER_VERSION="$(get_remote_version docker-pi-hole "${docker_branch}")" addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_DOCKER_VERSION" "${GITHUB_DOCKER_VERSION}" fi From 70376c24bdcb113a72652ae5add0e550811c242f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jun 2024 10:01:49 +0000 Subject: [PATCH 364/462] Bump eps1lon/actions-label-merge-conflict from 3.0.1 to 3.0.2 Bumps [eps1lon/actions-label-merge-conflict](https://github.com/eps1lon/actions-label-merge-conflict) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/eps1lon/actions-label-merge-conflict/releases) - [Changelog](https://github.com/eps1lon/actions-label-merge-conflict/blob/main/CHANGELOG.md) - [Commits](https://github.com/eps1lon/actions-label-merge-conflict/compare/v3.0.1...v3.0.2) --- updated-dependencies: - dependency-name: eps1lon/actions-label-merge-conflict dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/merge-conflict.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index 491ede68..4e56ae3e 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v3.0.1 + uses: eps1lon/actions-label-merge-conflict@v3.0.2 with: dirtyLabel: "PR: Merge Conflict" repoToken: "${{ secrets.GITHUB_TOKEN }}" From 4add164d43bfbaee81c563857b93bfb350d5a32b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jun 2024 10:20:46 +0000 Subject: [PATCH 365/462] Bump eps1lon/actions-label-merge-conflict from 3.0.1 to 3.0.2 Bumps [eps1lon/actions-label-merge-conflict](https://github.com/eps1lon/actions-label-merge-conflict) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/eps1lon/actions-label-merge-conflict/releases) - [Changelog](https://github.com/eps1lon/actions-label-merge-conflict/blob/main/CHANGELOG.md) - [Commits](https://github.com/eps1lon/actions-label-merge-conflict/compare/v3.0.1...v3.0.2) --- updated-dependencies: - dependency-name: eps1lon/actions-label-merge-conflict dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/merge-conflict.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/merge-conflict.yml b/.github/workflows/merge-conflict.yml index 491ede68..4e56ae3e 100644 --- a/.github/workflows/merge-conflict.yml +++ b/.github/workflows/merge-conflict.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check if PRs are have merge conflicts - uses: eps1lon/actions-label-merge-conflict@v3.0.1 + uses: eps1lon/actions-label-merge-conflict@v3.0.2 with: dirtyLabel: "PR: Merge Conflict" repoToken: "${{ secrets.GITHUB_TOKEN }}" From 628e1bbe413774c16422f944f1f80eaadae4220f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jun 2024 10:22:49 +0000 Subject: [PATCH 366/462] Bump pytest-testinfra from 10.1.0 to 10.1.1 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 10.1.0 to 10.1.1. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/10.1.0...10.1.1) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 3ece8551..90c15130 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 8.2.1 pytest-xdist == 3.6.1 -pytest-testinfra == 10.1.0 +pytest-testinfra == 10.1.1 tox == 4.15.0 From 36f05cb20f1ef7fbcc543f1c41dc49ddba9928e6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Jun 2024 10:29:56 +0000 Subject: [PATCH 367/462] Bump pytest-testinfra from 10.1.0 to 10.1.1 in /test Bumps [pytest-testinfra](https://github.com/pytest-dev/pytest-testinfra) from 10.1.0 to 10.1.1. - [Release notes](https://github.com/pytest-dev/pytest-testinfra/releases) - [Changelog](https://github.com/pytest-dev/pytest-testinfra/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest-testinfra/compare/10.1.0...10.1.1) --- updated-dependencies: - dependency-name: pytest-testinfra dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 3ece8551..90c15130 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.1 pytest == 8.2.1 pytest-xdist == 3.6.1 -pytest-testinfra == 10.1.0 +pytest-testinfra == 10.1.1 tox == 4.15.0 From e232361b2d779dc42bd08fa4ffb436a8f2989a21 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 7 Jun 2024 19:02:43 +0200 Subject: [PATCH 368/462] Add CAP_SYS_TIME to FTL's ambient capabilities Signed-off-by: DL6ER --- advanced/Templates/pihole-FTL.service | 2 +- advanced/Templates/pihole-FTL.systemd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 460339ae..009401fc 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -37,7 +37,7 @@ start() { # Run pre-start script, which pre-creates all expected files with correct permissions sh "${PI_HOLE_SCRIPT_DIR}/pihole-FTL-prestart.sh" - if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_IPC_LOCK,CAP_CHOWN+eip "/usr/bin/pihole-FTL"; then + if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_IPC_LOCK,CAP_CHOWN,CAP_SYS_TIME+eip "/usr/bin/pihole-FTL"; then su -s /bin/sh -c "/usr/bin/pihole-FTL" pihole else echo "Warning: Starting pihole-FTL as root because setting capabilities is not supported on this system" diff --git a/advanced/Templates/pihole-FTL.systemd b/advanced/Templates/pihole-FTL.systemd index 2a114199..0a3d270e 100644 --- a/advanced/Templates/pihole-FTL.systemd +++ b/advanced/Templates/pihole-FTL.systemd @@ -18,7 +18,7 @@ StartLimitIntervalSec=60s [Service] User=pihole PermissionsStartOnly=true -AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_NET_RAW CAP_NET_ADMIN CAP_SYS_NICE CAP_IPC_LOCK CAP_CHOWN +AmbientCapabilities=CAP_NET_BIND_SERVICE CAP_NET_RAW CAP_NET_ADMIN CAP_SYS_NICE CAP_IPC_LOCK CAP_CHOWN CAP_SYS_TIME ExecStartPre=/opt/pihole/pihole-FTL-prestart.sh ExecStart=/usr/bin/pihole-FTL -f From 2fdb6559dcfb0b036b9aef4d9a64e363965053cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 8 Jun 2024 10:15:56 +0000 Subject: [PATCH 369/462] Bump tox from 4.15.0 to 4.15.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.15.0 to 4.15.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.15.0...4.15.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 90c15130..5cfd7e68 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.2.1 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.15.0 +tox == 4.15.1 From 0742c017a894cc4c69f14c02fa6e55c107468f4e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 8 Jun 2024 10:57:29 +0000 Subject: [PATCH 370/462] Bump pytest from 8.2.1 to 8.2.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.2.1 to 8.2.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.2.1...8.2.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 5cfd7e68..557d6497 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.2.1 +pytest == 8.2.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 tox == 4.15.1 From cfc0d0717342342fa5f45df2ec29bc0f3a39685e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 8 Jun 2024 12:57:45 +0200 Subject: [PATCH 371/462] Remove CentOS8 from test suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 1 - test/_centos_8.Dockerfile | 18 ------------------ test/tox.centos_8.ini | 8 -------- 3 files changed, 27 deletions(-) delete mode 100644 test/_centos_8.Dockerfile delete mode 100644 test/tox.centos_8.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 568c4669..5a5ee85f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,7 +63,6 @@ jobs: ubuntu_22, ubuntu_23, ubuntu_24, - centos_8, centos_9, fedora_39, fedora_40, diff --git a/test/_centos_8.Dockerfile b/test/_centos_8.Dockerfile deleted file mode 100644 index a07a67e9..00000000 --- a/test/_centos_8.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM quay.io/centos/centos:stream8 -RUN yum install -y git initscripts - -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole - -RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole -ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR - -RUN true && \ - chmod +x $SCRIPTDIR/* - -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net - -#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.centos_8.ini b/test/tox.centos_8.ini deleted file mode 100644 index 85ae1ffb..00000000 --- a/test/tox.centos_8.ini +++ /dev/null @@ -1,8 +0,0 @@ -[tox] -envlist = py3 - -[testenv:py3] -allowlist_externals = docker -deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _centos_8.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py From fdd5b5ced0872704289d5a48d18a9ec54e22147d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Jun 2024 10:14:34 +0000 Subject: [PATCH 372/462] Bump actions/checkout from 4.1.6 to 4.1.7 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.6 to 4.1.7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.6...v4.1.7) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index dd97f2ee..0a5b59b9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 3e9ccbc3..2218ea02 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index d215e80f..fe24c9b5 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 92f95320..d7f3aea4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Check scripts in repository are executable run: | @@ -65,7 +65,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From 5d2cb552d3e6309330eef6a02cd5d99b83f36247 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 15 Jun 2024 10:25:08 +0000 Subject: [PATCH 373/462] Bump actions/checkout from 4.1.6 to 4.1.7 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.6 to 4.1.7. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.6...v4.1.7) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index dd97f2ee..0a5b59b9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 3e9ccbc3..2218ea02 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index d215e80f..fe24c9b5 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5a5ee85f..1da17f0a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Check scripts in repository are executable run: | @@ -71,7 +71,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.6 + uses: actions/checkout@v4.1.7 - name: Set up Python 3.10 uses: actions/setup-python@v5.1.0 From 520641fa5e26e54f92d53d8e4c4d16d2b702fbfd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:18:11 +0200 Subject: [PATCH 374/462] Try to use the CLI password for logins (if enabled and readable by the current user) Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 12 +++++++++--- advanced/Scripts/query.sh | 16 ++++------------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 18a48ce7..efffa25a 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -75,9 +75,15 @@ TestAPIAvailability() { } Authentication() { - # Try to authenticate - LoginAPI + # Try to read the CLI password (if enabled and readable by the current user) + if [ -r /etc/pihole/cli_pw ]; then + password=$(cat /etc/pihole/cli_pw) + # Try to authenticate using the CLI password + LoginAPI + fi + + # If this did not work, ask the user for the password while [ "${validSession}" = false ] || [ -z "${validSession}" ] ; do echo "Authentication failed. Please enter your Pi-hole password" @@ -105,7 +111,7 @@ LoginAPI() { SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) } -DeleteSession() { +LogoutAPI() { # if a valid Session exists (no password required or successful Authentication) and # SID is not null (successful Authentication only), delete the session if [ "${validSession}" = true ] && [ ! "${SID}" = null ]; then diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 493c75ea..123eee21 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -128,22 +128,14 @@ Main() { # Test if the authentication endpoint is available TestAPIAvailability - # Users can configure FTL in a way, that for accessing a) all endpoints (webserver.api.localAPIauth) - # or b) for the /search endpoint (webserver.api.searchAPIauth) no authentication is required. - # Therefore, we try to query directly without authentication but do authenticat if 401 is returned + # Authenticate with FTL + Authentication + # send query again data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") - if [ "${data}" = 401 ]; then - # Unauthenticated, so authenticate with the FTL server required - Authentication - - # send query again - data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") - fi - GenerateOutput "${data}" - DeleteSession + LogoutAPI } # Process all options (if present) From a2951cd3b8c220d09e7e70a24a747cda1a9bf129 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:19:54 +0200 Subject: [PATCH 375/462] Hide successful login/logout messages to avoid cluttering the terminal Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index efffa25a..3d35742f 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -94,9 +94,6 @@ Authentication() { LoginAPI done - # Loop exited, authentication was successful - echo "Authentication successful." - } LoginAPI() { @@ -119,7 +116,6 @@ LogoutAPI() { deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in - "204") printf "%b" "Session successfully deleted.\n";; "401") printf "%b" "Logout attempt without a valid session. Unauthorized!\n";; esac; fi From 4df7cee6c20bf3bbcf3ad806345f3b8dc82aced3 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:21:43 +0200 Subject: [PATCH 376/462] Add partial matching hint if exact matching found nothing Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 4 ++-- advanced/Scripts/query.sh | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 3d35742f..4162eff1 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -74,7 +74,7 @@ TestAPIAvailability() { fi } -Authentication() { +LoginAPI() { # Try to read the CLI password (if enabled and readable by the current user) if [ -r /etc/pihole/cli_pw ]; then password=$(cat /etc/pihole/cli_pw) @@ -96,7 +96,7 @@ Authentication() { } -LoginAPI() { +Authentication() { sessionResponse="$(curl -skS -X POST "${API_URL}auth" --user-agent "Pi-hole cli " --data "{\"password\":\"${password}\"}" )" if [ -z "${sessionResponse}" ]; then diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 123eee21..c76e890e 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -112,6 +112,12 @@ GenerateOutput() { printf "\n\n" done fi + + # If no exact results were found, suggest using partial matching + if [ "${num_lists}" -eq 0 ] && [ "${num_gravity}" -eq 0 ] && [ "${partial}" = false ]; then + printf "%s\n" "Hint: Try partial matching with" + printf "%s\n\n" " ${COL_GREEN}pihole -q --partial ${domain}${COL_NC}" + fi } Main() { @@ -129,12 +135,14 @@ Main() { TestAPIAvailability # Authenticate with FTL - Authentication + LoginAPI # send query again data=$(GetFTLData "search/${domain}?N=${max_results}&partial=${partial}") GenerateOutput "${data}" + + # Delete session LogoutAPI } From 5cb9f4faaafd2fdb78c3e56b454ea65ea8687e05 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:28:14 +0200 Subject: [PATCH 377/462] Modify pihole -f to use TOML config items Signed-off-by: DL6ER --- advanced/Scripts/piholeLogFlush.sh | 44 +++++++++++++++++------------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 4d97fec5..892645af 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -11,27 +11,29 @@ colfile="/opt/pihole/COL_TABLE" source ${colfile} +readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" +utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +source "${utilsfile}" + # In case we're running at the same time as a system logrotate, use a # separate logrotate state file to prevent stepping on each other's # toes. STATEFILE="/var/lib/logrotate/pihole" # Determine database location -# Obtain DBFILE=... setting from pihole-FTL.db -# Constructed to return nothing when -# a) the setting is not present in the config file, or -# b) the setting is commented out (e.g. "#DBFILE=...") -FTLconf="/etc/pihole/pihole-FTL.conf" -if [ -e "$FTLconf" ]; then - DBFILE="$(sed -n -e 's/^\s*DBFILE\s*=\s*//p' ${FTLconf})" -fi -# Test for empty string. Use standard path in this case. +DBFILE=$(getFTLConfigValue "files.database") if [ -z "$DBFILE" ]; then DBFILE="/etc/pihole/pihole-FTL.db" fi +# Determine log file location +LOGFILE=$(getFTLConfigValue "files.log.dnsmasq") +if [ -z "$LOGFILE" ]; then + LOGFILE="/var/log/pihole.log" +fi + if [[ "$*" != *"quiet"* ]]; then - echo -ne " ${INFO} Flushing /var/log/pihole/pihole.log ..." + echo -ne " ${INFO} Flushing "${LOGFILE}" ..." fi if [[ "$*" == *"once"* ]]; then # Nightly logrotation @@ -44,9 +46,9 @@ if [[ "$*" == *"once"* ]]; then # Note that moving the file is not an option, as # dnsmasq would happily continue writing into the # moved file (it will have the same file handler) - cp -p /var/log/pihole/pihole.log /var/log/pihole/pihole.log.1 - echo " " > /var/log/pihole/pihole.log - chmod 640 /var/log/pihole/pihole.log + cp -p "${LOGFILE}" "${LOGFILE}.1" + echo " " > "${LOGFILE}" + chmod 640 "${LOGFILE}" fi else # Manual flushing @@ -56,17 +58,21 @@ else /usr/sbin/logrotate --force --state "${STATEFILE}" /etc/pihole/logrotate else # Flush both pihole.log and pihole.log.1 (if existing) - echo " " > /var/log/pihole/pihole.log - if [ -f /var/log/pihole/pihole.log.1 ]; then - echo " " > /var/log/pihole/pihole.log.1 - chmod 640 /var/log/pihole/pihole.log.1 + echo " " > "${LOGFILE}" + if [ -f "${LOGFILE}.1" ]; then + echo " " > "${LOGFILE}.1" + chmod 640 "${LOGFILE}.1" fi fi + + # Stop FTL to make sure it doesn't write to the database while we're deleting data + service pihole-FTL stop + # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) deleted=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") - # Restart pihole-FTL to force reloading history - sudo pihole restartdns + # Restart FTL + service pihole-FTL restart fi if [[ "$*" != *"quiet"* ]]; then From 8f24e8aa5f67e5386cd36f87bb6f9ab9d6d34e9a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:41:42 +0200 Subject: [PATCH 378/462] Modify pihole -t to use TOML config items Signed-off-by: DL6ER --- pihole | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pihole b/pihole index 7c84771c..f7963d73 100755 --- a/pihole +++ b/pihole @@ -391,19 +391,20 @@ exit 0 tailFunc() { # Warn user if Pi-hole's logging is disabled - local logging_enabled=$(grep -c "^log-queries" /etc/dnsmasq.d/01-pihole.conf) - if [[ "${logging_enabled}" == "0" ]]; then - # No "log-queries" lines are found. - # Commented out lines (such as "#log-queries") are ignored + local logging_enabled=$(getFTLConfigValue dns.queryLogging) + if [[ "${logging_enabled}" != "true" ]]; then echo " ${CROSS} Warning: Query logging is disabled" fi echo -e " ${INFO} Press Ctrl-C to exit" + # Get logfile path + readonly LOGFILE=$(getFTLConfigValue files.log.dnsmasq) + # Strip date from each line # Color blocklist/blacklist/wildcard entries as red # Color A/AAAA/DHCP strings as white # Color everything else as gray - tail -f /var/log/pihole/pihole.log | grep --line-buffered "${1}" | sed -E \ + tail -f $LOGFILE | grep --line-buffered "${1}" | sed -E \ -e "s,($(date +'%b %d ')| dnsmasq\[[0-9]*\]),,g" \ -e "s,(.*(blacklisted |gravity blocked ).*),${COL_RED}&${COL_NC}," \ -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \ From 92b15cf7441462fceefeb0cd936845b0a5be984a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 22:49:11 +0200 Subject: [PATCH 379/462] Modify pihole arpflush to stop FTL while performing the action and use the new TOML config values Signed-off-by: DL6ER --- advanced/Scripts/piholeARPTable.sh | 36 +++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index b92dd124..c04c5b33 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -15,27 +15,29 @@ if [[ -f ${coltable} ]]; then source ${coltable} fi +readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" +utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +source "${utilsfile}" + # Determine database location -# Obtain DBFILE=... setting from pihole-FTL.db -# Constructed to return nothing when -# a) the setting is not present in the config file, or -# b) the setting is commented out (e.g. "#DBFILE=...") -FTLconf="/etc/pihole/pihole-FTL.conf" -if [ -e "$FTLconf" ]; then - DBFILE="$(sed -n -e 's/^\s*DBFILE\s*=\s*//p' ${FTLconf})" -fi -# Test for empty string. Use standard path in this case. +DBFILE=$(getFTLConfigValue "files.database") if [ -z "$DBFILE" ]; then DBFILE="/etc/pihole/pihole-FTL.db" fi - flushARP(){ local output if [[ "${args[1]}" != "quiet" ]]; then echo -ne " ${INFO} Flushing network table ..." fi + # Stop FTL to prevent database access + if ! output=$(pihole-FTL service stop 2>&1); then + echo -e "${OVER} ${CROSS} Failed to stop FTL" + echo " Output: ${output}" + return 1 + fi + # Truncate network_addresses table in pihole-FTL.db # This needs to be done before we can truncate the network table due to # foreign key constraints @@ -54,6 +56,20 @@ flushARP(){ return 1 fi + # Flush ARP cache of the host + if ! output=$(ip -s -s neigh flush all 2>&1); then + echo -e "${OVER} ${CROSS} Failed to flush ARP cache" + echo " Output: ${output}" + return 1 + fi + + # Start FTL again + if ! output=$(pihole-FTL service restart 2>&1); then + echo -e "${OVER} ${CROSS} Failed to restart FTL" + echo " Output: ${output}" + return 1 + fi + if [[ "${args[1]}" != "quiet" ]]; then echo -e "${OVER} ${TICK} Flushed network table" fi From bfc18f8329ad6eca2d4c2e3ea9e641d86844a453 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 19 Jun 2024 23:04:39 +0200 Subject: [PATCH 380/462] Rewrite list functions to use the API Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 18 +- advanced/Scripts/list.sh | 358 +++++++++++++++------------------------ pihole | 20 +-- 3 files changed, 164 insertions(+), 232 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 4162eff1..21447105 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -75,12 +75,16 @@ TestAPIAvailability() { } LoginAPI() { + if [ -z "${API_URL}" ]; then + TestAPIAvailability + fi + # Try to read the CLI password (if enabled and readable by the current user) if [ -r /etc/pihole/cli_pw ]; then password=$(cat /etc/pihole/cli_pw) # Try to authenticate using the CLI password - LoginAPI + Authentication fi # If this did not work, ask the user for the password @@ -91,7 +95,7 @@ LoginAPI() { secretRead; printf '\n' # Try to authenticate again - LoginAPI + Authentication done } @@ -144,6 +148,16 @@ GetFTLData() { fi } +PostFTLData() { + local data response status + # send the data to the API + response=$(curl -skS -w "%{http_code}" -X POST "${API_URL}$1" --data-raw "$2" -H "Accept: application/json" -H "sid: ${SID}" ) + # status are the last 3 characters + status=$(printf %s "${response#"${response%???}"}") + # data is everything from response without the last 3 characters + printf %s "${response%???}" +} + secretRead() { # POSIX compliant function to read user-input and diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 76558e58..3bd4af75 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -5,261 +5,187 @@ # (c) 2017 Pi-hole, LLC (https://pi-hole.net) # Network-wide ad blocking via your own hardware. # -# Whitelist and blacklist domains +# allowlist and denylist domains # # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -# Globals -piholeDir="/etc/pihole" -GRAVITYDB="${piholeDir}/gravity.db" -# Source pihole-FTL from install script -pihole_FTL="${piholeDir}/pihole-FTL.conf" -if [[ -f "${pihole_FTL}" ]]; then - source "${pihole_FTL}" +readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" +readonly utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +source "${utilsfile}" + +readonly apifile="${PI_HOLE_SCRIPT_DIR}/api.sh" +source "${apifile}" + +# Determine database location +DBFILE=$(getFTLConfigValue "files.database") +if [ -z "$DBFILE" ]; then + DBFILE="/etc/pihole/pihole-FTL.db" fi -# Set this only after sourcing pihole-FTL.conf as the gravity database path may -# have changed -gravityDBfile="${GRAVITYDB}" +# Determine gravity database location +GRAVITYDB=$(getFTLConfigValue "files.gravity") +if [ -z "$GRAVITYDB" ]; then + GRAVITYDB="/etc/pihole/gravity.db" +fi -noReloadRequested=false addmode=true verbose=true wildcard=false -web=false domList=() typeId="" comment="" -declare -i domaincount -domaincount=0 -reload=false colfile="/opt/pihole/COL_TABLE" source ${colfile} -# IDs are hard-wired to domain interpretation in the gravity database scheme -# Clients (including FTL) will read them through the corresponding views -readonly whitelist="0" -readonly blacklist="1" -readonly regex_whitelist="2" -readonly regex_blacklist="3" - -GetListnameFromTypeId() { - if [[ "$1" == "${whitelist}" ]]; then - echo "whitelist" - elif [[ "$1" == "${blacklist}" ]]; then - echo "blacklist" - elif [[ "$1" == "${regex_whitelist}" ]]; then - echo "regex whitelist" - elif [[ "$1" == "${regex_blacklist}" ]]; then - echo "regex blacklist" - fi -} - -GetListParamFromTypeId() { - if [[ "${typeId}" == "${whitelist}" ]]; then - echo "w" - elif [[ "${typeId}" == "${blacklist}" ]]; then - echo "b" - elif [[ "${typeId}" == "${regex_whitelist}" && "${wildcard}" == true ]]; then - echo "-white-wild" - elif [[ "${typeId}" == "${regex_whitelist}" ]]; then - echo "-white-regex" - elif [[ "${typeId}" == "${regex_blacklist}" && "${wildcard}" == true ]]; then - echo "-wild" - elif [[ "${typeId}" == "${regex_blacklist}" ]]; then - echo "-regex" - fi -} - helpFunc() { - local listname param - - listname="$(GetListnameFromTypeId "${typeId}")" - param="$(GetListParamFromTypeId)" - - echo "Usage: pihole -${param} [options] -Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com' -${listname^} one or more domains + echo "Usage: pihole ${abbrv} [options] +Example: 'pihole ${abbrv} site.com', or 'pihole ${abbrv} site1.com site2.com' +${typeId^} one or more ${kindId} domains Options: - -d, --delmode Remove domain(s) from the ${listname} - -nr, --noreload Update ${listname} without reloading the DNS server + -d, --delmode Remove domain(s) -q, --quiet Make output less verbose -h, --help Show this help dialog - -l, --list Display all your ${listname}listed domains + -l, --list Display domains --nuke Removes all entries in a list --comment \"text\" Add a comment to the domain. If adding multiple domains the same comment will be used for all" exit 0 } -ValidateDomain() { - # Convert to lowercase - domain="${1,,}" - local str validDomain - - # Check validity of domain (don't check for regex entries) - if [[ ( "${typeId}" == "${regex_blacklist}" || "${typeId}" == "${regex_whitelist}" ) && "${wildcard}" == false ]]; then - validDomain="${domain}" - else - # Check max length - if [[ "${#domain}" -le 253 ]]; then - validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check - validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label - # set error string - str="is not a valid argument or domain name!" - else - validDomain= - str="is too long!" - - fi +CreateDomainList() { + # Format domain into regex filter if requested + local dom=${1} + if [[ "${wildcard}" == true ]]; then + dom="(\\.|^)${dom//\./\\.}$" fi - - if [[ -n "${validDomain}" ]]; then - domList=("${domList[@]}" "${validDomain}") - else - echo -e " ${CROSS} ${domain} ${str}" - fi - - domaincount=$((domaincount+1)) -} - -ProcessDomainList() { - for dom in "${domList[@]}"; do - # Format domain into regex filter if requested - if [[ "${wildcard}" == true ]]; then - dom="(\\.|^)${dom//\./\\.}$" - fi - - # Logic: If addmode then add to desired list and remove from the other; - # if delmode then remove from desired list but do not add to the other - if ${addmode}; then - AddDomain "${dom}" - else - RemoveDomain "${dom}" - fi - done + domList=("${domList[@]}" "${dom}") } AddDomain() { - local domain num requestedListname existingTypeId existingListname - domain="$1" + local json num - # Is the domain in the list we want to add it to? - num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")" - requestedListname="$(GetListnameFromTypeId "${typeId}")" + # Authenticate with the API + LoginAPI - if [[ "${num}" -ne 0 ]]; then - existingTypeId="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")" - if [[ "${existingTypeId}" == "${typeId}" ]]; then - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${requestedListname}, no need to add!" + # Prepare request to POST /api/domains/{type}/{kind} + # Build JSON object of the following form + # { + # "domain": [ ], + # "comment": + # } + # where is an array of domain strings and is a string + # We use jq to build the JSON object + json=$(jq --null-input --compact-output --arg domains "${domList[*]}" --arg comment "${comment}" '{domain: $domains | split(" "), comment: $comment}') + + # Send the request + data=$(PostFTLData "domains/${typeId}/${kindId}" "${json}") + + # Display domain(s) added + # (they are listed in .processed.success, use jq) + num=$(echo "${data}" | jq '.processed.success | length') + if [[ "${num}" -gt 0 ]] && [[ "${verbose}" == true ]]; then + echo -e " ${TICK} Added ${num} domain(s):" + for i in $(seq 0 $((num-1))); do + echo -e " - ${COL_BLUE}$(echo "${data}" | jq --raw-output ".processed.success[$i].item")${COL_NC}" + done + fi + # Display failed domain(s) + # (they are listed in .processed.errors, use jq) + num=$(echo "${data}" | jq '.processed.errors | length') + if [[ "${num}" -gt 0 ]] && [[ "${verbose}" == true ]]; then + echo -e " ${CROSS} Failed to add ${num} domain(s):" + for i in $(seq 0 $((num-1))); do + echo -e " - ${COL_BLUE}$(echo "${data}" | jq --raw-output ".processed.errors[$i].item")${COL_NC}" + error=$(echo "${data}" | jq --raw-output ".processed.errors[$i].error") + if [[ "${error}" == "UNIQUE constraint failed: domainlist.domain, domainlist.type" ]]; then + error="Domain already in the specified list" fi - else - existingListname="$(GetListnameFromTypeId "${existingTypeId}")" - pihole-FTL sqlite3 -ni "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';" - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!" - fi - fi - return + echo -e " ${error}" + done fi - # Domain not found in the table, add it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${domain} to the ${requestedListname}..." - fi - reload=true - # Insert only the domain here. The enabled and date_added fields will be filled - # with their default values (enabled = true, date_added = current timestamp) - if [[ -z "${comment}" ]]; then - pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});" - else - # also add comment when variable has been set through the "--comment" option - pihole-FTL sqlite3 -ni "${gravityDBfile}" "INSERT INTO domainlist (domain,type,comment) VALUES ('${domain}',${typeId},'${comment}');" - fi + # Log out + LogoutAPI } RemoveDomain() { - local domain num requestedListname - domain="$1" + local json num - # Is the domain in the list we want to remove it from? - num="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")" + # Authenticate with the API + LoginAPI - requestedListname="$(GetListnameFromTypeId "${typeId}")" + # Prepare request to POST /api/domains:batchDelete + # Build JSON object of the following form + # [{ + # "item": , + # "type": "${typeId}", + # "kind": "${kindId}", + # }] + # where is the domain string and ${typeId} and ${kindId} are the type and kind IDs + # We use jq to build the JSON object) + json=$(jq --null-input --compact-output --arg domains "${domList[*]}" --arg typeId "${typeId}" --arg kindId "${kindId}" '[ $domains | split(" ")[] as $item | {item: $item, type: $typeId, kind: $kindId} ]') - if [[ "${num}" -eq 0 ]]; then - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${domain} does not exist in ${requestedListname}, no need to remove!" - fi - return + # Send the request + data=$(PostFTLData "domains:batchDelete" "${json}") + + # If there is an .error object in the returned data, display it + local error + error=$(jq --compact-output <<< "${data}" '.error') + if [[ $error != "null" && $error != "" ]]; then + echo -e " ${CROSS} Failed to remove domain(s):" + echo -e " $(jq <<< "${data}" '.error')" + elif [[ "${verbose}" == true ]]; then + echo -e " ${TICK} Removed ${#domList[@]} domain(s):" + # Loop through the domains and display them + for dom in "${domList[@]}"; do + echo -e " - ${COL_BLUE}${dom}${COL_NC}" + done fi - # Domain found in the table, remove it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Removing ${domain} from the ${requestedListname}..." - fi - reload=true - # Remove it from the current list - pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};" + # Log out + LogoutAPI } Displaylist() { - local count num_pipes domain enabled status nicedate requestedListname + local data - requestedListname="$(GetListnameFromTypeId "${typeId}")" - data="$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)" - - if [[ -z $data ]]; then - echo -e "Not showing empty list" - else - echo -e "Displaying ${requestedListname}:" - count=1 - while IFS= read -r line - do - # Count number of pipes seen in this line - # This is necessary because we can only detect the pipe separating the fields - # from the end backwards as the domain (which is the first field) may contain - # pipe symbols as they are perfectly valid regex filter control characters - num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")" - - # Extract domain and enabled status based on the obtained number of pipe characters - domain="$(cut -d'|' -f"-$((num_pipes-1))" <<< "${line}")" - enabled="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")" - datemod="$(cut -d'|' -f"$((num_pipes+1))" <<< "${line}")" - - # Translate boolean status into human readable string - if [[ "${enabled}" -eq 1 ]]; then - status="enabled" - else - status="disabled" - fi - - # Get nice representation of numerical date stored in database - nicedate=$(date --rfc-2822 -d "@${datemod}") - - echo " ${count}: ${domain} (${status}, last modified ${nicedate})" - count=$((count+1)) - done <<< "${data}" + # if either typeId or kindId is empty, we cannot display the list + if [[ -z "${typeId}" ]] || [[ -z "${kindId}" ]]; then + echo " ${CROSS} Unable to display list. Please specify a list type and kind." + exit 1 fi - exit 0; -} -NukeList() { - count=$(pihole-FTL sqlite3 -ni "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") - listname="$(GetListnameFromTypeId "${typeId}")" - if [ "$count" -gt 0 ];then - pihole-FTL sqlite3 -ni "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" - echo " ${TICK} Removed ${count} domain(s) from the ${listname}" + # Authenticate with the API + LoginAPI + + # Send the request + data=$(GetFTLData "domains/${typeId}/${kindId}") + + # Display the list + num=$(echo "${data}" | jq '.domains | length') + if [[ "${num}" -gt 0 ]]; then + echo -e " ${TICK} Found ${num} domain(s) in the ${kindId} ${typeId}list:" + for i in $(seq 0 $((num-1))); do + echo -e " - ${COL_BLUE}$(echo "${data}" | jq --compact-output ".domains[$i].domain")${COL_NC}" + echo -e " Comment: $(echo "${data}" | jq --compact-output ".domains[$i].comment")" + echo -e " Groups: $(echo "${data}" | jq --compact-output ".domains[$i].groups")" + echo -e " Added: $(date -d @"$(echo "${data}" | jq --compact-output ".domains[$i].date_added")")" + echo -e " Last modified: $(date -d @"$(echo "${data}" | jq --compact-output ".domains[$i].date_modified")")" + done else - echo " ${INFO} ${listname} already empty. Nothing to do!" + echo -e " ${INFO} No domains found in the ${kindId} ${typeId}list" fi - exit 0; + + # Log out + LogoutAPI + + # Return early without adding/deleting domains + exit 0 } GetComment() { @@ -272,38 +198,30 @@ GetComment() { while (( "$#" )); do case "${1}" in - "-w" | "whitelist" ) typeId=0;; - "-b" | "blacklist" ) typeId=1;; - "--white-regex" | "white-regex" ) typeId=2;; - "--white-wild" | "white-wild" ) typeId=2; wildcard=true;; - "--wild" | "wildcard" ) typeId=3; wildcard=true;; - "--regex" | "regex" ) typeId=3;; - "-nr"| "--noreload" ) noReloadRequested=true;; + "-a" | "allowlist" ) kindId="exact"; typeId="allow"; abbrv="-a";; + "-b" | "denylist" ) kindId="exact"; typeId="deny"; abbrv="-b";; + "--allow-regex" | "allow-regex" ) kindId="regex"; typeId="allow"; abbrv="--allow-regex";; + "--allow-wild" | "allow-wild" ) kindId="regex"; typeId="allow"; wildcard=true; abbrv="--allow-wild";; + "--regex" | "regex" ) kindId="regex"; typeId="deny"; abbrv="--regex";; + "--wild" | "wildcard" ) kindId="regex"; typeId="deny"; wildcard=true; abbrv="--wild";; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; "-h" | "--help" ) helpFunc;; "-l" | "--list" ) Displaylist;; - "--nuke" ) NukeList;; - "--web" ) web=true;; "--comment" ) GetComment "${2}"; shift;; - * ) ValidateDomain "${1}";; + * ) CreateDomainList "${1}";; esac shift done shift -if [[ ${domaincount} == 0 ]]; then +if [[ ${#domList[@]} == 0 ]]; then helpFunc fi -ProcessDomainList - -# Used on web interface -if $web; then - echo "DONE" -fi - -if [[ ${reload} == true && ${noReloadRequested} == false ]]; then - pihole restartdns reload-lists +if ${addmode}; then + AddDomain +else + RemoveDomain fi diff --git a/pihole b/pihole index f7963d73..ce46fd0f 100755 --- a/pihole +++ b/pihole @@ -537,12 +537,12 @@ case "${1}" in "tricorder" ) tricorderFunc;; # we need to add all arguments that require sudo power to not trigger the * argument - "-w" | "whitelist" ) ;; - "-b" | "blacklist" ) ;; - "--wild" | "wildcard" ) ;; - "--regex" | "regex" ) ;; - "--white-regex" | "white-regex" ) ;; - "--white-wild" | "white-wild" ) ;; + "-a" | "allowlist" ) need_root=0;; + "-b" | "blocklist" | "denylist" ) need_root=0;; + "--wild" | "wildcard" ) need_root=0;; + "--regex" | "regex" ) need_root=0;; + "--allow-regex" | "allow-regex" ) need_root=0;; + "--allow-wild" | "allow-wild" ) need_root=0;; "-f" | "flush" ) ;; "-up" | "updatePihole" ) ;; "-r" | "reconfigure" ) ;; @@ -592,12 +592,12 @@ fi # Handle redirecting to specific functions based on arguments case "${1}" in - "-w" | "whitelist" ) listFunc "$@";; - "-b" | "blacklist" ) listFunc "$@";; + "-a" | "allowlist" ) listFunc "$@";; + "-b" | "blocklist" | "denylist" ) listFunc "$@";; "--wild" | "wildcard" ) listFunc "$@";; "--regex" | "regex" ) listFunc "$@";; - "--white-regex" | "white-regex" ) listFunc "$@";; - "--white-wild" | "white-wild" ) listFunc "$@";; + "--allow-regex" | "allow-regex" ) listFunc "$@";; + "--allow-wild" | "allow-wild" ) listFunc "$@";; "-d" | "debug" ) debugFunc "$@";; "-f" | "flush" ) flushFunc "$@";; "-up" | "updatePihole" ) updatePiholeFunc "$@";; From 25f384a923c3f0feccedc3ee4033cc482f2683c8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 20 Jun 2024 19:41:47 +0200 Subject: [PATCH 381/462] Do not use CLI password when no password is set at all Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 21447105..6f31a6d2 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -51,6 +51,12 @@ TestAPIAvailability() { API_PORT="" else # API is available at this URL combination + + if [ "${availabilityResonse}" = 200 ]; then + # API is available without authentication + needAuth=false + fi + break fi @@ -75,10 +81,16 @@ TestAPIAvailability() { } LoginAPI() { + # If the API URL is not set, test the availability if [ -z "${API_URL}" ]; then TestAPIAvailability fi + # Exit early if authentication is not needed + if [ "${needAuth}" = false ]; then + return + fi + # Try to read the CLI password (if enabled and readable by the current user) if [ -r /etc/pihole/cli_pw ]; then password=$(cat /etc/pihole/cli_pw) @@ -87,6 +99,8 @@ LoginAPI() { Authentication fi + + # If this did not work, ask the user for the password while [ "${validSession}" = false ] || [ -z "${validSession}" ] ; do echo "Authentication failed. Please enter your Pi-hole password" From 5dfcd02c40b9babd4f824805d53d4d919116a944 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 22 Jun 2024 10:31:03 +0200 Subject: [PATCH 382/462] Improve pihole -f Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 8 +-- advanced/Scripts/piholeLogFlush.sh | 79 +++++++++++++++++++++++------- 2 files changed, 64 insertions(+), 23 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 6f31a6d2..ea057adc 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -21,7 +21,7 @@ TestAPIAvailability() { # as we are running locally, we can get the port value from FTL directly - local chaos_api_list availabilityResonse + local chaos_api_list availabilityResponse # Query the API URLs from FTL using CHAOS TXT local.api.ftl # The result is a space-separated enumeration of full URLs @@ -43,16 +43,16 @@ TestAPIAvailability() { API_URL="${API_URL#\"}" # Test if the API is available at this URL - availabilityResonse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") + availabilityResponse=$(curl -skS -o /dev/null -w "%{http_code}" "${API_URL}auth") # Test if http status code was 200 (OK) or 401 (authentication required) - if [ ! "${availabilityResonse}" = 200 ] && [ ! "${availabilityResonse}" = 401 ]; then + if [ ! "${availabilityResponse}" = 200 ] && [ ! "${availabilityResponse}" = 401 ]; then # API is not available at this port/protocol combination API_PORT="" else # API is available at this URL combination - if [ "${availabilityResonse}" = 200 ]; then + if [ "${availabilityResponse}" = 200 ]; then # API is available without authentication needAuth=false fi diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 892645af..34d96318 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -29,16 +29,21 @@ fi # Determine log file location LOGFILE=$(getFTLConfigValue "files.log.dnsmasq") if [ -z "$LOGFILE" ]; then - LOGFILE="/var/log/pihole.log" + LOGFILE="/var/log/pihole/pihole.log" +fi +FTLFILE=$(getFTLConfigValue "files.log.ftl") +if [ -z "$FTLFILE" ]; then + FTLFILE="/var/log/pihole/FTL.log" fi -if [[ "$*" != *"quiet"* ]]; then - echo -ne " ${INFO} Flushing "${LOGFILE}" ..." -fi if [[ "$*" == *"once"* ]]; then # Nightly logrotation if command -v /usr/sbin/logrotate >/dev/null; then # Logrotate once + + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Running logrotate ..." + fi /usr/sbin/logrotate --force --state "${STATEFILE}" /etc/pihole/logrotate else # Copy pihole.log over to pihole.log.1 @@ -46,23 +51,60 @@ if [[ "$*" == *"once"* ]]; then # Note that moving the file is not an option, as # dnsmasq would happily continue writing into the # moved file (it will have the same file handler) + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Rotating ${LOGFILE} ..." + fi cp -p "${LOGFILE}" "${LOGFILE}.1" echo " " > "${LOGFILE}" chmod 640 "${LOGFILE}" + if [[ "$*" != *"quiet"* ]]; then + echo -e "${OVER} ${TICK} Rotated ${LOGFILE} ..." + fi + # Copy FTL.log over to FTL.log.1 + # and empty out FTL.log + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Rotating ${FTLFILE} ..." + fi + cp -p "${FTLFILE}" "${FTLFILE}.1" + echo " " > "${FTLFILE}" + chmod 640 "${FTLFILE}" + if [[ "$*" != *"quiet"* ]]; then + echo -e "${OVER} ${TICK} Rotated ${FTLFILE} ..." + fi fi else # Manual flushing - if command -v /usr/sbin/logrotate >/dev/null; then - # Logrotate twice to move all data out of sight of FTL - /usr/sbin/logrotate --force --state "${STATEFILE}" /etc/pihole/logrotate; sleep 3 - /usr/sbin/logrotate --force --state "${STATEFILE}" /etc/pihole/logrotate - else - # Flush both pihole.log and pihole.log.1 (if existing) - echo " " > "${LOGFILE}" - if [ -f "${LOGFILE}.1" ]; then - echo " " > "${LOGFILE}.1" - chmod 640 "${LOGFILE}.1" - fi + + # Flush both pihole.log and pihole.log.1 (if existing) + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Flushing ${LOGFILE} ..." + fi + echo " " > "${LOGFILE}" + chmod 640 "${LOGFILE}" + if [ -f "${LOGFILE}.1" ]; then + echo " " > "${LOGFILE}.1" + chmod 640 "${LOGFILE}.1" + fi + if [[ "$*" != *"quiet"* ]]; then + echo -e "${OVER} ${TICK} Flushed ${LOGFILE} ..." + fi + + # Flush both FTL.log and FTL.log.1 (if existing) + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Flushing ${FTLFILE} ..." + fi + echo " " > "${FTLFILE}" + chmod 640 "${FTLFILE}" + if [ -f "${FTLFILE}.1" ]; then + echo " " > "${FTLFILE}.1" + chmod 640 "${FTLFILE}.1" + fi + if [[ "$*" != *"quiet"* ]]; then + echo -e "${OVER} ${TICK} Flushed ${FTLFILE} ..." + fi + + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Flushing database, DNS resolution temporarily unavailable ..." fi # Stop FTL to make sure it doesn't write to the database while we're deleting data @@ -73,9 +115,8 @@ else # Restart FTL service pihole-FTL restart + if [[ "$*" != *"quiet"* ]]; then + echo -e "${OVER} ${TICK} Deleted ${deleted} queries from long-term query database" + fi fi -if [[ "$*" != *"quiet"* ]]; then - echo -e "${OVER} ${TICK} Flushed /var/log/pihole/pihole.log" - echo -e " ${TICK} Deleted ${deleted} queries from database" -fi From 97324ae97e3b10e4456d3499f0e8ef5dabbc878b Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 22 Jun 2024 13:49:27 +0100 Subject: [PATCH 383/462] remove centos8 tests from development to prevent build fails (yes, I know it's likely that we will merge development-v6 -> development" soon" but.. just in case) Signed-off-by: Adam Warner --- .github/workflows/test.yml | 1 - test/_centos_8.Dockerfile | 18 ------------------ test/tox.centos_8.ini | 8 -------- 3 files changed, 27 deletions(-) delete mode 100644 test/_centos_8.Dockerfile delete mode 100644 test/tox.centos_8.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7f3aea4..ec2b5728 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,7 +56,6 @@ jobs: ubuntu_22, ubuntu_23, ubuntu_24, - centos_8, centos_9, fedora_39, fedora_40, diff --git a/test/_centos_8.Dockerfile b/test/_centos_8.Dockerfile deleted file mode 100644 index a07a67e9..00000000 --- a/test/_centos_8.Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM quay.io/centos/centos:stream8 -RUN yum install -y git initscripts - -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole - -RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole -ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR - -RUN true && \ - chmod +x $SCRIPTDIR/* - -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net - -#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.centos_8.ini b/test/tox.centos_8.ini deleted file mode 100644 index dca77c93..00000000 --- a/test/tox.centos_8.ini +++ /dev/null @@ -1,8 +0,0 @@ -[tox] -envlist = py3 - -[testenv:py3] -allowlist_externals = docker -deps = -rrequirements.txt -commands = docker buildx build --load --progress plain -f _centos_8.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py From 7e91b9ab47f710311ed9cc2d509c5ea3a4c5405e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 22 Jun 2024 20:35:06 +0200 Subject: [PATCH 384/462] Update help text Signed-off-by: DL6ER --- pihole | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pihole b/pihole index ce46fd0f..d9a22c63 100755 --- a/pihole +++ b/pihole @@ -472,17 +472,17 @@ unsupportedFunc(){ helpFunc() { echo "Usage: pihole [options] -Example: 'pihole -w -h' +Example: 'pihole -a -h' Add '-h' after specific commands for more information on usage -Whitelist/Blacklist Options: - -w, whitelist Whitelist domain(s) - -b, blacklist Blacklist domain(s) +Domain Options: + -a, allowlist Allowlist domain(s) + -b, denylist Denylist domain(s) --regex, regex Regex blacklist domains(s) - --white-regex Regex whitelist domains(s) + --allow-regex Regex allowlist domains(s) --wild, wildcard Wildcard blacklist domain(s) - --white-wild Wildcard whitelist domain(s) - Add '-h' for more info on whitelist/blacklist usage + --allow-wild Wildcard allowlist domain(s) + Add '-h' for more info on allowlist/denylist usage Debugging Options: -d, debug Start a debugging session From 424e825bd9a28a416766067dc22821ae81bd139f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 22 Jun 2024 20:41:19 +0200 Subject: [PATCH 385/462] Do not auto-sudo in the pihole command Signed-off-by: DL6ER --- pihole | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/pihole b/pihole index d9a22c63..cae0cfb2 100755 --- a/pihole +++ b/pihole @@ -552,8 +552,8 @@ case "${1}" in "disable" ) ;; "-d" | "debug" ) ;; "restartdns" ) ;; - "-g" | "updateGravity" ) need_root=0;; - "reloaddns" ) need_root=0;; + "-g" | "updateGravity" ) ;; + "reloaddns" ) ;; "setpassword" ) ;; "checkout" ) ;; "updatechecker" ) ;; @@ -562,32 +562,18 @@ case "${1}" in * ) helpFunc;; esac -# Must be root to use this tool for most functions -if [[ ! $EUID -eq 0 && need_root -eq 1 ]];then - if [[ -x "$(command -v sudo)" ]]; then - exec sudo bash "$0" "$@" - exit $? - else - echo -e " ${CROSS} sudo is needed to run pihole commands. Please run this script as root or install sudo." - exit 1 - fi -fi - # In the case of alpine running in a container, the USER variable appears to be blank # which prevents the next trap from working correctly. Set it by running whoami if [[ -z ${USER} ]]; then USER=$(whoami) fi -# Can also be user pihole for other functions -if [[ ${USER} != "pihole" && need_root -eq 0 ]];then - if [[ -x "$(command -v sudo)" ]]; then - exec sudo -u pihole bash "$0" "$@" - exit $? - else - echo -e " ${CROSS} sudo is needed to run pihole commands. Please run this script as root or install sudo." - exit 1 - fi +# Check if the current user is neither root nor pihole and if the command +# requires root. If so, exit with an error message. +if [[ $EUID -ne 0 && ${USER} != "pihole" && need_root -eq 1 ]];then + echo -e " ${CROSS} The Pi-hole command requires root privileges, try:" + echo -e " ${COL_GREEN}sudo pihole ${@}${COL_NC}" + exit 1 fi # Handle redirecting to specific functions based on arguments From b835fa06a6360004c9fa75e56225fba06b7374f0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 22 Jun 2024 20:48:07 +0200 Subject: [PATCH 386/462] Further black -> deny renaming Signed-off-by: DL6ER --- pihole | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pihole b/pihole index cae0cfb2..00cbd893 100755 --- a/pihole +++ b/pihole @@ -401,12 +401,12 @@ tailFunc() { readonly LOGFILE=$(getFTLConfigValue files.log.dnsmasq) # Strip date from each line - # Color blocklist/blacklist/wildcard entries as red + # Color blocklist/denylist/wildcard entries as red # Color A/AAAA/DHCP strings as white # Color everything else as gray tail -f $LOGFILE | grep --line-buffered "${1}" | sed -E \ -e "s,($(date +'%b %d ')| dnsmasq\[[0-9]*\]),,g" \ - -e "s,(.*(blacklisted |gravity blocked ).*),${COL_RED}&${COL_NC}," \ + -e "s,(.*(denied |gravity blocked ).*),${COL_RED}&${COL_NC}," \ -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \ -e "s,.*,${COL_GRAY}&${COL_NC}," exit 0 @@ -478,9 +478,9 @@ Add '-h' after specific commands for more information on usage Domain Options: -a, allowlist Allowlist domain(s) -b, denylist Denylist domain(s) - --regex, regex Regex blacklist domains(s) + --regex, regex Regex denylist domains(s) --allow-regex Regex allowlist domains(s) - --wild, wildcard Wildcard blacklist domain(s) + --wild, wildcard Wildcard denylist domain(s) --allow-wild Wildcard allowlist domain(s) Add '-h' for more info on allowlist/denylist usage From fe8e63853cf69e077eae5dedbe523f19ec0a5b0b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 22 Jun 2024 20:49:44 +0200 Subject: [PATCH 387/462] Use concatenate of arguments instead of using the array directly Signed-off-by: DL6ER --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index 00cbd893..1313d487 100755 --- a/pihole +++ b/pihole @@ -572,7 +572,7 @@ fi # requires root. If so, exit with an error message. if [[ $EUID -ne 0 && ${USER} != "pihole" && need_root -eq 1 ]];then echo -e " ${CROSS} The Pi-hole command requires root privileges, try:" - echo -e " ${COL_GREEN}sudo pihole ${@}${COL_NC}" + echo -e " ${COL_GREEN}sudo pihole $*${COL_NC}" exit 1 fi From bfc2cf69e519222863399115d44814647d47b72e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 Jun 2024 16:01:05 +0000 Subject: [PATCH 388/462] Bump tox from 4.15.0 to 4.15.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.15.0 to 4.15.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.15.0...4.15.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 90c15130..5cfd7e68 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.2.1 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.15.0 +tox == 4.15.1 From 548e1424064354682772f3a68dd8fa44ce30c4b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 22 Jun 2024 16:00:48 +0000 Subject: [PATCH 389/462] Bump pytest from 8.2.1 to 8.2.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.2.1 to 8.2.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.2.1...8.2.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 5cfd7e68..557d6497 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.2.1 +pytest == 8.2.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 tox == 4.15.1 From ccdbfd41309346ebb955e8910f2c130b92cd60d8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Jun 2024 13:56:49 +0200 Subject: [PATCH 390/462] Use natural langauge for list manipulations, like pihole allow example.com or pihole deny other.net. Also remove using pihole deny not bad.org Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 9 ++++--- advanced/bash-completion/pihole | 6 ++--- manpages/pihole.8 | 42 +++++++++++++++------------------ pihole | 25 ++++++++++---------- 4 files changed, 39 insertions(+), 43 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 3bd4af75..144317ce 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -47,11 +47,10 @@ Example: 'pihole ${abbrv} site.com', or 'pihole ${abbrv} site1.com site2.com' ${typeId^} one or more ${kindId} domains Options: - -d, --delmode Remove domain(s) + not -d, --delmode Remove domain(s) -q, --quiet Make output less verbose -h, --help Show this help dialog -l, --list Display domains - --nuke Removes all entries in a list --comment \"text\" Add a comment to the domain. If adding multiple domains the same comment will be used for all" exit 0 @@ -198,13 +197,13 @@ GetComment() { while (( "$#" )); do case "${1}" in - "-a" | "allowlist" ) kindId="exact"; typeId="allow"; abbrv="-a";; - "-b" | "denylist" ) kindId="exact"; typeId="deny"; abbrv="-b";; + "allow" | "allowlist" ) kindId="exact"; typeId="allow"; abbrv="allow";; + "deny" | "denylist" ) kindId="exact"; typeId="deny"; abbrv="deny";; "--allow-regex" | "allow-regex" ) kindId="regex"; typeId="allow"; abbrv="--allow-regex";; "--allow-wild" | "allow-wild" ) kindId="regex"; typeId="allow"; wildcard=true; abbrv="--allow-wild";; "--regex" | "regex" ) kindId="regex"; typeId="deny"; abbrv="--regex";; "--wild" | "wildcard" ) kindId="regex"; typeId="deny"; wildcard=true; abbrv="--wild";; - "-d" | "--delmode" ) addmode=false;; + "-d" | "not" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; "-h" | "--help" ) helpFunc;; "-l" | "--list" ) Displaylist;; diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 89e02d2f..064193b4 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -7,11 +7,11 @@ _pihole() { case "${prev}" in "pihole") - opts="blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" + opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard arpflush" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; - "whitelist"|"blacklist"|"wildcard"|"regex") - opts_lists="\--delmode \--noreload \--quiet \--list \--nuke" + "allow"|"deny"|"wildcard"|"regex"|"allow-regx"|"allow-wild") + opts_lists="\not \--delmode \--quiet \--list \--help" COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) ;; "checkout") diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 55bbe6cb..1bf917db 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -52,47 +52,43 @@ pihole restartdns\fR [options] Available commands and options: .br -\fB-w, whitelist\fR [options] [ ] +\fBallow, allowlist\fR [options] [ ] .br - Adds or removes specified domain or domains to the Whitelist + Adds or removes specified domain or domains to the Allowlist .br -\fB-b, blacklist\fR [options] [ ] +\fBdeny, denylist\fR [options] [ ] .br - Adds or removes specified domain or domains to the blacklist + Adds or removes specified domain or domains to the denylist .br \fB--regex, regex\fR [options] [ ] .br - Add or removes specified regex filter to the regex blacklist + Add or removes specified regex filter to the regex denylist .br -\fB--white-regex\fR [options] [ ] +\fB--allow-regex\fR [options] [ ] .br - Add or removes specified regex filter to the regex whitelist + Add or removes specified regex filter to the regex allowlist .br \fB--wild, wildcard\fR [options] [ ] .br - Add or removes specified domain to the wildcard blacklist + Add or removes specified domain to the wildcard denylist .br -\fB--white-wild\fR [options] [ ] +\fB--allow-wild\fR [options] [ ] .br - Add or removes specified domain to the wildcard whitelist + Add or removes specified domain to the wildcard allowlist .br - (Whitelist/Blacklist manipulation options): + (Allow-/denylist manipulation options): .br - -d, --delmode Remove domain(s) from the list + not, -d, --delmode Remove domain(s) from the list .br - -nr, --noreload Update list without refreshing dnsmasq + -q, --quiet Make output less verbose .br - -q, --quiet Make output less verbose -.br - -l, --list Display all your listed domains -.br - --nuke Removes all entries in a list + -l, --list Display all your listed domains .br \fB-d, debug\fR [-a] @@ -279,17 +275,17 @@ Available commands and options: Some usage examples .br -Whitelist/blacklist manipulation +Allow-/denylist manipulation .br -\fBpihole -w iloveads.example.com\fR +\fBpihole allow iloveads.example.com\fR .br - Adds "iloveads.example.com" to whitelist + Allow "iloveads.example.com" .br -\fBpihole -b -d noads.example.com\fR +\fBpihole deny not noads.example.com\fR .br - Removes "noads.example.com" from blacklist + Removes "noads.example.com" from denylist .br \fBpihole --wild example.com\fR diff --git a/pihole b/pihole index 1313d487..5a3c847d 100755 --- a/pihole +++ b/pihole @@ -472,17 +472,17 @@ unsupportedFunc(){ helpFunc() { echo "Usage: pihole [options] -Example: 'pihole -a -h' +Example: 'pihole allow -h' Add '-h' after specific commands for more information on usage Domain Options: - -a, allowlist Allowlist domain(s) - -b, denylist Denylist domain(s) - --regex, regex Regex denylist domains(s) - --allow-regex Regex allowlist domains(s) - --wild, wildcard Wildcard denylist domain(s) - --allow-wild Wildcard allowlist domain(s) - Add '-h' for more info on allowlist/denylist usage + allow, allowlist Allow domain(s) + deny, denylist Deny domain(s) + --regex, regex Regex deny domains(s) + --allow-regex Regex allow domains(s) + --wild, wildcard Wildcard deny domain(s) + --allow-wild Wildcard allow domain(s) + Add '-h' for more info on allow/deny usage Debugging Options: -d, debug Start a debugging session @@ -537,8 +537,8 @@ case "${1}" in "tricorder" ) tricorderFunc;; # we need to add all arguments that require sudo power to not trigger the * argument - "-a" | "allowlist" ) need_root=0;; - "-b" | "blocklist" | "denylist" ) need_root=0;; + "allow" | "allowlist" ) need_root=0;; + "deny" | "denylist" ) need_root=0;; "--wild" | "wildcard" ) need_root=0;; "--regex" | "regex" ) need_root=0;; "--allow-regex" | "allow-regex" ) need_root=0;; @@ -578,8 +578,8 @@ fi # Handle redirecting to specific functions based on arguments case "${1}" in - "-a" | "allowlist" ) listFunc "$@";; - "-b" | "blocklist" | "denylist" ) listFunc "$@";; + "allow" | "allowlist" ) listFunc "$@";; + "deny" | "denylist" ) listFunc "$@";; "--wild" | "wildcard" ) listFunc "$@";; "--regex" | "regex" ) listFunc "$@";; "--allow-regex" | "allow-regex" ) listFunc "$@";; @@ -600,4 +600,5 @@ case "${1}" in "updatechecker" ) shift; updateCheckFunc "$@";; "arpflush" ) arpFunc "$@";; "-t" | "tail" ) tailFunc "$2";; + * ) helpFunc;; esac From c2ed30480deabaf5d53db6bcc617807ef52f6034 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 25 Jun 2024 14:10:35 +0200 Subject: [PATCH 391/462] Show when requested domains were not found on the list Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 10 +++++++--- advanced/Scripts/list.sh | 19 ++++++++++--------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index ea057adc..5843c585 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -166,10 +166,14 @@ PostFTLData() { local data response status # send the data to the API response=$(curl -skS -w "%{http_code}" -X POST "${API_URL}$1" --data-raw "$2" -H "Accept: application/json" -H "sid: ${SID}" ) - # status are the last 3 characters - status=$(printf %s "${response#"${response%???}"}") # data is everything from response without the last 3 characters - printf %s "${response%???}" + if [ "${3}" = "status" ]; then + # Keep the status code appended if requested + printf %s "${response}" + else + # Strip the status code + printf %s "${response%???}" + fi } secretRead() { diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 144317ce..d6a30325 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -66,7 +66,7 @@ CreateDomainList() { } AddDomain() { - local json num + local json num data # Authenticate with the API LoginAPI @@ -113,7 +113,7 @@ AddDomain() { } RemoveDomain() { - local json num + local json num data status # Authenticate with the API LoginAPI @@ -130,7 +130,10 @@ RemoveDomain() { json=$(jq --null-input --compact-output --arg domains "${domList[*]}" --arg typeId "${typeId}" --arg kindId "${kindId}" '[ $domains | split(" ")[] as $item | {item: $item, type: $typeId, kind: $kindId} ]') # Send the request - data=$(PostFTLData "domains:batchDelete" "${json}") + data=$(PostFTLData "domains:batchDelete" "${json}" "status") + # Separate the status from the data + status=$(printf %s "${data#"${data%???}"}") + data=$(printf %s "${data%???}") # If there is an .error object in the returned data, display it local error @@ -138,12 +141,10 @@ RemoveDomain() { if [[ $error != "null" && $error != "" ]]; then echo -e " ${CROSS} Failed to remove domain(s):" echo -e " $(jq <<< "${data}" '.error')" - elif [[ "${verbose}" == true ]]; then - echo -e " ${TICK} Removed ${#domList[@]} domain(s):" - # Loop through the domains and display them - for dom in "${domList[@]}"; do - echo -e " - ${COL_BLUE}${dom}${COL_NC}" - done + elif [[ "${verbose}" == true && "${status}" == "204" ]]; then + echo -e " ${TICK} Domain(s) removed from the ${kindId} ${typeId}list" + elif [[ "${verbose}" == true && "${status}" == "404" ]]; then + echo -e " ${TICK} Requested domain(s) not found on ${kindId} ${typeId}list" fi # Log out From 21fb5dabe19f97c53a76c9b0b81034352df5ce82 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 29 Jun 2024 09:32:13 +0200 Subject: [PATCH 392/462] Use "remove" and "delete" instead of "not" to trigger listed domain removals Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index d6a30325..8c930f04 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -47,7 +47,7 @@ Example: 'pihole ${abbrv} site.com', or 'pihole ${abbrv} site1.com site2.com' ${typeId^} one or more ${kindId} domains Options: - not -d, --delmode Remove domain(s) + remove, delete, -d Remove domain(s) -q, --quiet Make output less verbose -h, --help Show this help dialog -l, --list Display domains @@ -204,7 +204,7 @@ while (( "$#" )); do "--allow-wild" | "allow-wild" ) kindId="regex"; typeId="allow"; wildcard=true; abbrv="--allow-wild";; "--regex" | "regex" ) kindId="regex"; typeId="deny"; abbrv="--regex";; "--wild" | "wildcard" ) kindId="regex"; typeId="deny"; wildcard=true; abbrv="--wild";; - "-d" | "not" | "--delmode" ) addmode=false;; + "-d" | "remove" | "delete" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; "-h" | "--help" ) helpFunc;; "-l" | "--list" ) Displaylist;; From aa704a2e0e5644e8f188a55f31bc514293275ba5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 1 Jul 2024 21:07:09 +0200 Subject: [PATCH 393/462] Add pytest-clarity to test environment to improve error log output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 557d6497..f68261de 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -3,4 +3,4 @@ pytest == 8.2.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 tox == 4.15.1 - +pytest-clarity == 1.0.1 From 6b1d0e09d1786c27c273ad1d172d664c7ade6fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 2 Jul 2024 08:23:48 +0200 Subject: [PATCH 394/462] Set column width to 120 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/tox.centos_9.ini | 2 ++ test/tox.debian_10.ini | 2 ++ test/tox.debian_11.ini | 2 ++ test/tox.debian_12.ini | 2 ++ test/tox.fedora_39.ini | 2 ++ test/tox.fedora_40.ini | 2 ++ test/tox.ubuntu_20.ini | 2 ++ test/tox.ubuntu_22.ini | 2 ++ test/tox.ubuntu_23.ini | 2 ++ test/tox.ubuntu_24.ini | 2 ++ 10 files changed, 20 insertions(+) diff --git a/test/tox.centos_9.ini b/test/tox.centos_9.ini index 4c51aefa..81dd0bd2 100644 --- a/test/tox.centos_9.ini +++ b/test/tox.centos_9.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _centos_9.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.debian_10.ini b/test/tox.debian_10.ini index f107300f..9995a852 100644 --- a/test/tox.debian_10.ini +++ b/test/tox.debian_10.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _debian_10.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.debian_11.ini b/test/tox.debian_11.ini index c38a15fb..a8909d46 100644 --- a/test/tox.debian_11.ini +++ b/test/tox.debian_11.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _debian_11.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.debian_12.ini b/test/tox.debian_12.ini index ee70e8bd..707e8710 100644 --- a/test/tox.debian_12.ini +++ b/test/tox.debian_12.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _debian_12.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.fedora_39.ini b/test/tox.fedora_39.ini index 5c8557c9..aaa6b30e 100644 --- a/test/tox.fedora_39.ini +++ b/test/tox.fedora_39.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _fedora_39.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.fedora_40.ini b/test/tox.fedora_40.ini index 149630d7..462c5ff1 100644 --- a/test/tox.fedora_40.ini +++ b/test/tox.fedora_40.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _fedora_40.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.ubuntu_20.ini b/test/tox.ubuntu_20.ini index 49a6153e..bcfb1d2a 100644 --- a/test/tox.ubuntu_20.ini +++ b/test/tox.ubuntu_20.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _ubuntu_20.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.ubuntu_22.ini b/test/tox.ubuntu_22.ini index 8014d6d6..c8e71abb 100644 --- a/test/tox.ubuntu_22.ini +++ b/test/tox.ubuntu_22.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _ubuntu_22.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.ubuntu_23.ini b/test/tox.ubuntu_23.ini index 767ed9ef..f0a32a68 100644 --- a/test/tox.ubuntu_23.ini +++ b/test/tox.ubuntu_23.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _ubuntu_23.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py diff --git a/test/tox.ubuntu_24.ini b/test/tox.ubuntu_24.ini index dbd278d6..5b7e77a9 100644 --- a/test/tox.ubuntu_24.ini +++ b/test/tox.ubuntu_24.ini @@ -4,5 +4,7 @@ envlist = py3 [testenv:py3] allowlist_externals = docker deps = -rrequirements.txt +setenv = + COLUMNS=120 commands = docker buildx build --load --progress plain -f _ubuntu_24.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py From ddc289ca3e66e7739a63d1e91a7d42606157eb23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 2 Jul 2024 20:45:34 +0200 Subject: [PATCH 395/462] Fix Dockerfile syntax ENV legacy warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/_centos_9.Dockerfile | 10 +++++----- test/_debian_10.Dockerfile | 10 +++++----- test/_debian_11.Dockerfile | 10 +++++----- test/_debian_12.Dockerfile | 10 +++++----- test/_fedora_39.Dockerfile | 10 +++++----- test/_fedora_40.Dockerfile | 10 +++++----- test/_ubuntu_20.Dockerfile | 11 +++++------ test/_ubuntu_22.Dockerfile | 10 +++++----- test/_ubuntu_23.Dockerfile | 10 +++++----- test/_ubuntu_24.Dockerfile | 10 +++++----- 10 files changed, 50 insertions(+), 51 deletions(-) diff --git a/test/_centos_9.Dockerfile b/test/_centos_9.Dockerfile index 6ccd18b6..2e3d055a 100644 --- a/test/_centos_9.Dockerfile +++ b/test/_centos_9.Dockerfile @@ -1,18 +1,18 @@ FROM quay.io/centos/centos:stream9 RUN yum install -y --allowerasing curl git initscripts -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_debian_10.Dockerfile b/test/_debian_10.Dockerfile index 3b177cc8..dc813ac2 100644 --- a/test/_debian_10.Dockerfile +++ b/test/_debian_10.Dockerfile @@ -1,17 +1,17 @@ FROM buildpack-deps:buster-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_debian_11.Dockerfile b/test/_debian_11.Dockerfile index 58c67e0f..cb7d27cc 100644 --- a/test/_debian_11.Dockerfile +++ b/test/_debian_11.Dockerfile @@ -1,17 +1,17 @@ FROM buildpack-deps:bullseye-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_debian_12.Dockerfile b/test/_debian_12.Dockerfile index a762fee0..50d709b1 100644 --- a/test/_debian_12.Dockerfile +++ b/test/_debian_12.Dockerfile @@ -1,17 +1,17 @@ FROM buildpack-deps:bookworm-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_fedora_39.Dockerfile b/test/_fedora_39.Dockerfile index 1727a3aa..1d3dbc63 100644 --- a/test/_fedora_39.Dockerfile +++ b/test/_fedora_39.Dockerfile @@ -1,18 +1,18 @@ FROM fedora:39 RUN dnf install -y git initscripts -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_fedora_40.Dockerfile b/test/_fedora_40.Dockerfile index 6d00072f..e4879c92 100644 --- a/test/_fedora_40.Dockerfile +++ b/test/_fedora_40.Dockerfile @@ -1,18 +1,18 @@ FROM fedora:40 RUN dnf install -y git initscripts -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_ubuntu_20.Dockerfile b/test/_ubuntu_20.Dockerfile index c63f883a..64d4f415 100644 --- a/test/_ubuntu_20.Dockerfile +++ b/test/_ubuntu_20.Dockerfile @@ -1,18 +1,17 @@ FROM buildpack-deps:focal-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR -ENV DEBIAN_FRONTEND=noninteractive +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_ubuntu_22.Dockerfile b/test/_ubuntu_22.Dockerfile index d44518b4..34faa361 100644 --- a/test/_ubuntu_22.Dockerfile +++ b/test/_ubuntu_22.Dockerfile @@ -1,18 +1,18 @@ FROM buildpack-deps:jammy-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR ENV DEBIAN_FRONTEND=noninteractive RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_ubuntu_23.Dockerfile b/test/_ubuntu_23.Dockerfile index f9b3910b..ea0ad245 100644 --- a/test/_ubuntu_23.Dockerfile +++ b/test/_ubuntu_23.Dockerfile @@ -1,18 +1,18 @@ FROM buildpack-deps:lunar-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR ENV DEBIAN_FRONTEND=noninteractive RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_ubuntu_24.Dockerfile b/test/_ubuntu_24.Dockerfile index 2b048361..4d581cd3 100644 --- a/test/_ubuntu_24.Dockerfile +++ b/test/_ubuntu_24.Dockerfile @@ -1,18 +1,18 @@ FROM buildpack-deps:24.04-scm -ENV GITDIR /etc/.pihole -ENV SCRIPTDIR /opt/pihole +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole ADD . $GITDIR RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR ENV DEBIAN_FRONTEND=noninteractive RUN true && \ chmod +x $SCRIPTDIR/* -ENV SKIP_INSTALL true -ENV OS_CHECK_DOMAIN_NAME dev-supportedos.pi-hole.net +ENV SKIP_INSTALL=true +ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ From fafd47ba75a95b6c663af811f0407b818ee7dcb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 2 Jul 2024 20:52:44 +0200 Subject: [PATCH 396/462] Please editorconfig-checker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/stale.yml | 1 - .../Scripts/database_migration/gravity-db.sh | 284 +++++++++--------- .../database_migration/gravity/1_to_2.sql | 6 +- .../database_migration/gravity/2_to_3.sql | 24 +- .../database_migration/gravity/3_to_4.sql | 20 +- .../database_migration/gravity/4_to_5.sql | 16 +- .../database_migration/gravity/5_to_6.sql | 1 - .../database_migration/gravity/7_to_8.sql | 12 +- advanced/Templates/gravity.db.sql | 100 +++--- advanced/Templates/logrotate | 48 +-- advanced/bash-completion/pihole | 94 +++--- manpages/pihole.8 | 2 +- 12 files changed, 303 insertions(+), 305 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 2218ea02..2a7831af 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -45,4 +45,3 @@ jobs: run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index e99f1df2..8f84e2b0 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -13,150 +13,150 @@ readonly scriptPath="/etc/.pihole/advanced/Scripts/database_migration/gravity" upgrade_gravityDB(){ - local database piholeDir auditFile version - database="${1}" - piholeDir="${2}" - auditFile="${piholeDir}/auditlog.list" + local database piholeDir auditFile version + database="${1}" + piholeDir="${2}" + auditFile="${piholeDir}/auditlog.list" - # Exit early if the database does not exist (e.g. in CI tests) - if [[ ! -f "${database}" ]]; then - return - fi + # Exit early if the database does not exist (e.g. in CI tests) + if [[ ! -f "${database}" ]]; then + return + fi - # Get database version - version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" + # Get database version + version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" - if [[ "$version" == "1" ]]; then - # This migration script upgrades the gravity.db file by - # adding the domain_audit table - echo -e " ${INFO} Upgrading gravity database from version 1 to 2" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql" - version=2 + if [[ "$version" == "1" ]]; then + # This migration script upgrades the gravity.db file by + # adding the domain_audit table + echo -e " ${INFO} Upgrading gravity database from version 1 to 2" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql" + version=2 - # Store audit domains in database table - if [ -e "${auditFile}" ]; then - echo -e " ${INFO} Migrating content of ${auditFile} into new database" - # database_table_from_file is defined in gravity.sh - database_table_from_file "domain_audit" "${auditFile}" - fi - fi - if [[ "$version" == "2" ]]; then - # This migration script upgrades the gravity.db file by - # renaming the regex table to regex_blacklist, and - # creating a new regex_whitelist table + corresponding linking table and views - echo -e " ${INFO} Upgrading gravity database from version 2 to 3" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql" - version=3 - fi - if [[ "$version" == "3" ]]; then - # This migration script unifies the formally separated domain - # lists into a single table with a UNIQUE domain constraint - echo -e " ${INFO} Upgrading gravity database from version 3 to 4" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql" - version=4 - fi - if [[ "$version" == "4" ]]; then - # This migration script upgrades the gravity and list views - # implementing necessary changes for per-client blocking - echo -e " ${INFO} Upgrading gravity database from version 4 to 5" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql" - version=5 - fi - if [[ "$version" == "5" ]]; then - # This migration script upgrades the adlist view - # to return an ID used in gravity.sh - echo -e " ${INFO} Upgrading gravity database from version 5 to 6" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql" - version=6 - fi - if [[ "$version" == "6" ]]; then - # This migration script adds a special group with ID 0 - # which is automatically associated to all clients not - # having their own group assignments - echo -e " ${INFO} Upgrading gravity database from version 6 to 7" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql" - version=7 - fi - if [[ "$version" == "7" ]]; then - # This migration script recreated the group table - # to ensure uniqueness on the group name - # We also add date_added and date_modified columns - echo -e " ${INFO} Upgrading gravity database from version 7 to 8" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql" - version=8 - fi - if [[ "$version" == "8" ]]; then - # This migration fixes some issues that were introduced - # in the previous migration script. - echo -e " ${INFO} Upgrading gravity database from version 8 to 9" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql" - version=9 - fi - if [[ "$version" == "9" ]]; then - # This migration drops unused tables and creates triggers to remove - # obsolete groups assignments when the linked items are deleted - echo -e " ${INFO} Upgrading gravity database from version 9 to 10" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql" - version=10 - fi - if [[ "$version" == "10" ]]; then - # This adds timestamp and an optional comment field to the client table - # These fields are only temporary and will be replaces by the columns - # defined in gravity.db.sql during gravity swapping. We add them here - # to keep the copying process generic (needs the same columns in both the - # source and the destination databases). - echo -e " ${INFO} Upgrading gravity database from version 10 to 11" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql" - version=11 - fi - if [[ "$version" == "11" ]]; then - # Rename group 0 from "Unassociated" to "Default" - echo -e " ${INFO} Upgrading gravity database from version 11 to 12" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql" - version=12 - fi - if [[ "$version" == "12" ]]; then - # Add column date_updated to adlist table - echo -e " ${INFO} Upgrading gravity database from version 12 to 13" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql" - version=13 - fi - if [[ "$version" == "13" ]]; then - # Add columns number and status to adlist table - echo -e " ${INFO} Upgrading gravity database from version 13 to 14" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql" - version=14 - fi - if [[ "$version" == "14" ]]; then - # Changes the vw_adlist created in 5_to_6 - echo -e " ${INFO} Upgrading gravity database from version 14 to 15" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql" - version=15 - fi - if [[ "$version" == "15" ]]; then - # Add column abp_entries to adlist table - echo -e " ${INFO} Upgrading gravity database from version 15 to 16" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/15_to_16.sql" - version=16 - fi - if [[ "$version" == "16" ]]; then - # Add antigravity table - # Add column type to adlist table (to support adlist types) - echo -e " ${INFO} Upgrading gravity database from version 16 to 17" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql" - version=17 - fi - if [[ "$version" == "17" ]]; then - # Add adlist.id to vw_gravity and vw_antigravity - echo -e " ${INFO} Upgrading gravity database from version 17 to 18" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql" - version=18 - fi - if [[ "$version" == "18" ]]; then - # Modify DELETE triggers to delete BEFORE instead of AFTER to prevent - # foreign key constraint violations - echo -e " ${INFO} Upgrading gravity database from version 18 to 19" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/18_to_19.sql" - version=19 - fi + # Store audit domains in database table + if [ -e "${auditFile}" ]; then + echo -e " ${INFO} Migrating content of ${auditFile} into new database" + # database_table_from_file is defined in gravity.sh + database_table_from_file "domain_audit" "${auditFile}" + fi + fi + if [[ "$version" == "2" ]]; then + # This migration script upgrades the gravity.db file by + # renaming the regex table to regex_blacklist, and + # creating a new regex_whitelist table + corresponding linking table and views + echo -e " ${INFO} Upgrading gravity database from version 2 to 3" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql" + version=3 + fi + if [[ "$version" == "3" ]]; then + # This migration script unifies the formally separated domain + # lists into a single table with a UNIQUE domain constraint + echo -e " ${INFO} Upgrading gravity database from version 3 to 4" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql" + version=4 + fi + if [[ "$version" == "4" ]]; then + # This migration script upgrades the gravity and list views + # implementing necessary changes for per-client blocking + echo -e " ${INFO} Upgrading gravity database from version 4 to 5" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql" + version=5 + fi + if [[ "$version" == "5" ]]; then + # This migration script upgrades the adlist view + # to return an ID used in gravity.sh + echo -e " ${INFO} Upgrading gravity database from version 5 to 6" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql" + version=6 + fi + if [[ "$version" == "6" ]]; then + # This migration script adds a special group with ID 0 + # which is automatically associated to all clients not + # having their own group assignments + echo -e " ${INFO} Upgrading gravity database from version 6 to 7" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql" + version=7 + fi + if [[ "$version" == "7" ]]; then + # This migration script recreated the group table + # to ensure uniqueness on the group name + # We also add date_added and date_modified columns + echo -e " ${INFO} Upgrading gravity database from version 7 to 8" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql" + version=8 + fi + if [[ "$version" == "8" ]]; then + # This migration fixes some issues that were introduced + # in the previous migration script. + echo -e " ${INFO} Upgrading gravity database from version 8 to 9" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql" + version=9 + fi + if [[ "$version" == "9" ]]; then + # This migration drops unused tables and creates triggers to remove + # obsolete groups assignments when the linked items are deleted + echo -e " ${INFO} Upgrading gravity database from version 9 to 10" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql" + version=10 + fi + if [[ "$version" == "10" ]]; then + # This adds timestamp and an optional comment field to the client table + # These fields are only temporary and will be replaces by the columns + # defined in gravity.db.sql during gravity swapping. We add them here + # to keep the copying process generic (needs the same columns in both the + # source and the destination databases). + echo -e " ${INFO} Upgrading gravity database from version 10 to 11" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql" + version=11 + fi + if [[ "$version" == "11" ]]; then + # Rename group 0 from "Unassociated" to "Default" + echo -e " ${INFO} Upgrading gravity database from version 11 to 12" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql" + version=12 + fi + if [[ "$version" == "12" ]]; then + # Add column date_updated to adlist table + echo -e " ${INFO} Upgrading gravity database from version 12 to 13" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql" + version=13 + fi + if [[ "$version" == "13" ]]; then + # Add columns number and status to adlist table + echo -e " ${INFO} Upgrading gravity database from version 13 to 14" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql" + version=14 + fi + if [[ "$version" == "14" ]]; then + # Changes the vw_adlist created in 5_to_6 + echo -e " ${INFO} Upgrading gravity database from version 14 to 15" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql" + version=15 + fi + if [[ "$version" == "15" ]]; then + # Add column abp_entries to adlist table + echo -e " ${INFO} Upgrading gravity database from version 15 to 16" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/15_to_16.sql" + version=16 + fi + if [[ "$version" == "16" ]]; then + # Add antigravity table + # Add column type to adlist table (to support adlist types) + echo -e " ${INFO} Upgrading gravity database from version 16 to 17" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/16_to_17.sql" + version=17 + fi + if [[ "$version" == "17" ]]; then + # Add adlist.id to vw_gravity and vw_antigravity + echo -e " ${INFO} Upgrading gravity database from version 17 to 18" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/17_to_18.sql" + version=18 + fi + if [[ "$version" == "18" ]]; then + # Modify DELETE triggers to delete BEFORE instead of AFTER to prevent + # foreign key constraint violations + echo -e " ${INFO} Upgrading gravity database from version 18 to 19" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/18_to_19.sql" + version=19 + fi } diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql index 6d57a6fe..ef445cc6 100644 --- a/advanced/Scripts/database_migration/gravity/1_to_2.sql +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -4,9 +4,9 @@ BEGIN TRANSACTION; CREATE TABLE domain_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) ); UPDATE info SET value = 2 WHERE property = 'version'; diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql index fd7c24d2..9ade340a 100644 --- a/advanced/Scripts/database_migration/gravity/2_to_3.sql +++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql @@ -8,9 +8,9 @@ ALTER TABLE regex RENAME TO regex_blacklist; CREATE TABLE regex_blacklist_by_group ( - regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (regex_blacklist_id, group_id) + regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_blacklist_id, group_id) ); INSERT INTO regex_blacklist_by_group SELECT * FROM regex_by_group; @@ -32,19 +32,19 @@ CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist CREATE TABLE regex_whitelist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT ); CREATE TABLE regex_whitelist_by_group ( - regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (regex_whitelist_id, group_id) + regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_whitelist_id, group_id) ); CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain diff --git a/advanced/Scripts/database_migration/gravity/3_to_4.sql b/advanced/Scripts/database_migration/gravity/3_to_4.sql index 05231f72..2b3d0a79 100644 --- a/advanced/Scripts/database_migration/gravity/3_to_4.sql +++ b/advanced/Scripts/database_migration/gravity/3_to_4.sql @@ -6,13 +6,13 @@ BEGIN TRANSACTION; CREATE TABLE domainlist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - type INTEGER NOT NULL DEFAULT 0, - domain TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + type INTEGER NOT NULL DEFAULT 0, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT ); ALTER TABLE whitelist ADD COLUMN type INTEGER; @@ -41,9 +41,9 @@ DROP TABLE regex_whitelist_by_group; DROP TABLE regex_blacklist_by_group; CREATE TABLE domainlist_by_group ( - domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (domainlist_id, group_id) + domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (domainlist_id, group_id) ); DROP TRIGGER tr_whitelist_update; diff --git a/advanced/Scripts/database_migration/gravity/4_to_5.sql b/advanced/Scripts/database_migration/gravity/4_to_5.sql index 4ae9f980..1436c69d 100644 --- a/advanced/Scripts/database_migration/gravity/4_to_5.sql +++ b/advanced/Scripts/database_migration/gravity/4_to_5.sql @@ -7,9 +7,9 @@ BEGIN TRANSACTION; DROP TABLE gravity; CREATE TABLE gravity ( - domain TEXT NOT NULL, - adlist_id INTEGER NOT NULL REFERENCES adlist (id), - PRIMARY KEY(domain, adlist_id) + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id), + PRIMARY KEY(domain, adlist_id) ); DROP VIEW vw_gravity; @@ -22,15 +22,15 @@ CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id CREATE TABLE client ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ip TEXT NOL NULL UNIQUE + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOL NULL UNIQUE ); CREATE TABLE client_by_group ( - client_id INTEGER NOT NULL REFERENCES client (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (client_id, group_id) + client_id INTEGER NOT NULL REFERENCES client (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (client_id, group_id) ); UPDATE info SET value = 5 WHERE property = 'version'; diff --git a/advanced/Scripts/database_migration/gravity/5_to_6.sql b/advanced/Scripts/database_migration/gravity/5_to_6.sql index d2bb3145..a058156a 100644 --- a/advanced/Scripts/database_migration/gravity/5_to_6.sql +++ b/advanced/Scripts/database_migration/gravity/5_to_6.sql @@ -15,4 +15,3 @@ CREATE VIEW vw_adlist AS SELECT DISTINCT address, adlist.id AS id UPDATE info SET value = 6 WHERE property = 'version'; COMMIT; - diff --git a/advanced/Scripts/database_migration/gravity/7_to_8.sql b/advanced/Scripts/database_migration/gravity/7_to_8.sql index ccf0c148..c6a5b35b 100644 --- a/advanced/Scripts/database_migration/gravity/7_to_8.sql +++ b/advanced/Scripts/database_migration/gravity/7_to_8.sql @@ -8,12 +8,12 @@ ALTER TABLE "group" RENAME TO "group__"; CREATE TABLE "group" ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - enabled BOOLEAN NOT NULL DEFAULT 1, - name TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - description TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + enabled BOOLEAN NOT NULL DEFAULT 1, + name TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + description TEXT ); CREATE TRIGGER tr_group_update AFTER UPDATE ON "group" diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 42060443..9782a044 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -3,99 +3,99 @@ BEGIN TRANSACTION; CREATE TABLE "group" ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - enabled BOOLEAN NOT NULL DEFAULT 1, - name TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - description TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + enabled BOOLEAN NOT NULL DEFAULT 1, + name TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + description TEXT ); INSERT INTO "group" (id,enabled,name,description) VALUES (0,1,'Default','The default group'); CREATE TABLE domainlist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - type INTEGER NOT NULL DEFAULT 0, - domain TEXT NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT, - UNIQUE(domain, type) + id INTEGER PRIMARY KEY AUTOINCREMENT, + type INTEGER NOT NULL DEFAULT 0, + domain TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT, + UNIQUE(domain, type) ); CREATE TABLE adlist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - address TEXT NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT, - date_updated INTEGER, - number INTEGER NOT NULL DEFAULT 0, - invalid_domains INTEGER NOT NULL DEFAULT 0, - status INTEGER NOT NULL DEFAULT 0, - abp_entries INTEGER NOT NULL DEFAULT 0, - type INTEGER NOT NULL DEFAULT 0, - UNIQUE(address, type) + id INTEGER PRIMARY KEY AUTOINCREMENT, + address TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT, + date_updated INTEGER, + number INTEGER NOT NULL DEFAULT 0, + invalid_domains INTEGER NOT NULL DEFAULT 0, + status INTEGER NOT NULL DEFAULT 0, + abp_entries INTEGER NOT NULL DEFAULT 0, + type INTEGER NOT NULL DEFAULT 0, + UNIQUE(address, type) ); CREATE TABLE adlist_by_group ( - adlist_id INTEGER NOT NULL REFERENCES adlist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (adlist_id, group_id) + adlist_id INTEGER NOT NULL REFERENCES adlist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (adlist_id, group_id) ); CREATE TABLE gravity ( - domain TEXT NOT NULL, - adlist_id INTEGER NOT NULL REFERENCES adlist (id) + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id) ); CREATE TABLE antigravity ( - domain TEXT NOT NULL, - adlist_id INTEGER NOT NULL REFERENCES adlist (id) + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id) ); CREATE TABLE info ( - property TEXT PRIMARY KEY, - value TEXT NOT NULL + property TEXT PRIMARY KEY, + value TEXT NOT NULL ); INSERT INTO "info" VALUES('version','18'); CREATE TABLE domain_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) ); CREATE TABLE domainlist_by_group ( - domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (domainlist_id, group_id) + domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (domainlist_id, group_id) ); CREATE TABLE client ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ip TEXT NOT NULL UNIQUE, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOT NULL UNIQUE, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT ); CREATE TABLE client_by_group ( - client_id INTEGER NOT NULL REFERENCES client (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (client_id, group_id) + client_id INTEGER NOT NULL REFERENCES client (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (client_id, group_id) ); CREATE TRIGGER tr_adlist_update AFTER UPDATE OF address,enabled,comment ON adlist diff --git a/advanced/Templates/logrotate b/advanced/Templates/logrotate index 5f609e0f..9e52776b 100644 --- a/advanced/Templates/logrotate +++ b/advanced/Templates/logrotate @@ -1,32 +1,32 @@ /var/log/pihole/pihole.log { - # su # - daily - copytruncate - rotate 5 - compress - delaycompress - notifempty - nomail + # su # + daily + copytruncate + rotate 5 + compress + delaycompress + notifempty + nomail } /var/log/pihole/FTL.log { - # su # - weekly - copytruncate - rotate 3 - compress - delaycompress - notifempty - nomail + # su # + weekly + copytruncate + rotate 3 + compress + delaycompress + notifempty + nomail } /var/log/pihole/webserver.log { - # su # - weekly - copytruncate - rotate 3 - compress - delaycompress - notifempty - nomail + # su # + weekly + copytruncate + rotate 3 + compress + delaycompress + notifempty + nomail } diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 89e02d2f..c2b58c6e 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -1,51 +1,51 @@ _pihole() { - local cur prev opts opts_checkout opts_debug opts_logging opts_query opts_update opts_version - COMPREPLY=() - cur="${COMP_WORDS[COMP_CWORD]}" - prev="${COMP_WORDS[COMP_CWORD-1]}" - prev2="${COMP_WORDS[COMP_CWORD-2]}" + local cur prev opts opts_checkout opts_debug opts_logging opts_query opts_update opts_version + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + prev2="${COMP_WORDS[COMP_CWORD-2]}" - case "${prev}" in - "pihole") - opts="blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" - COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) - ;; - "whitelist"|"blacklist"|"wildcard"|"regex") - opts_lists="\--delmode \--noreload \--quiet \--list \--nuke" - COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) - ;; - "checkout") - opts_checkout="core ftl web master dev" - COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) - ;; - "debug") - opts_debug="-a" - COMPREPLY=( $(compgen -W "${opts_debug}" -- ${cur}) ) - ;; - "logging") - opts_logging="on off 'off noflush'" - COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) ) - ;; - "query") - opts_query="--partial --all" - COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) ) - ;; - "updatePihole"|"-up") - opts_update="--check-only" - COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) - ;; - "core"|"admin"|"ftl") - if [[ "$prev2" == "checkout" ]]; then - opts_checkout="master dev" - COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) - else - return 1 - fi - ;; - *) - return 1 - ;; - esac - return 0 + case "${prev}" in + "pihole") + opts="blacklist checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + ;; + "whitelist"|"blacklist"|"wildcard"|"regex") + opts_lists="\--delmode \--noreload \--quiet \--list \--nuke" + COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) + ;; + "checkout") + opts_checkout="core ftl web master dev" + COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) + ;; + "debug") + opts_debug="-a" + COMPREPLY=( $(compgen -W "${opts_debug}" -- ${cur}) ) + ;; + "logging") + opts_logging="on off 'off noflush'" + COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) ) + ;; + "query") + opts_query="--partial --all" + COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) ) + ;; + "updatePihole"|"-up") + opts_update="--check-only" + COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) + ;; + "core"|"admin"|"ftl") + if [[ "$prev2" == "checkout" ]]; then + opts_checkout="master dev" + COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) + else + return 1 + fi + ;; + *) + return 1 + ;; + esac + return 0 } complete -F _pihole pihole diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 55bbe6cb..bdc4b744 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -170,7 +170,7 @@ Available commands and options: Specify whether the Pi-hole log should be used .br - (Logging options): + (Logging options): .br on Enable the Pi-hole log at /var/log/pihole/pihole.log .br From 2c497a9a3ea099079bbcd1eb21725b0ed54b529d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 30 Apr 2024 15:47:57 +0200 Subject: [PATCH 397/462] Add protocol validation when downloading blocklist from URL Signed-off-by: DL6ER --- gravity.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gravity.sh b/gravity.sh index 636cde0d..c4622ead 100755 --- a/gravity.sh +++ b/gravity.sh @@ -586,6 +586,12 @@ gravity_DownloadBlocklistFromUrl() { fi fi + # Check for allowed protocols + if [[ $url != "http"* && $url != "https"* && $url != "file"* && $url != "ftp"* && $url != "ftps"* && $url != "sftp"* ]]; then + echo -e "${OVER} ${CROSS} ${str} Invalid protocol specified, ignoring list" + download=false + fi + if [[ "${download}" == true ]]; then # shellcheck disable=SC2086 httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2> /dev/null) From 4148f2cb5b9f384e00976d1412c43c58037b1088 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Fri, 5 Jul 2024 19:03:40 +0100 Subject: [PATCH 398/462] tabs->spaces for editorconfig (Not sure how this has nto been caught before...) Signed-off-by: Adam Warner --- .../Scripts/database_migration/gravity-db.sh | 224 +++++++++--------- .../database_migration/gravity/1_to_2.sql | 6 +- .../database_migration/gravity/2_to_3.sql | 24 +- .../database_migration/gravity/3_to_4.sql | 20 +- .../database_migration/gravity/4_to_5.sql | 16 +- .../database_migration/gravity/7_to_8.sql | 12 +- advanced/Templates/gravity.db.sql | 90 +++---- advanced/Templates/logrotate | 32 +-- advanced/bash-completion/pihole | 150 ++++++------ manpages/pihole.8 | 2 +- 10 files changed, 288 insertions(+), 288 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 1459ecd9..378321ff 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -13,119 +13,119 @@ readonly scriptPath="/etc/.pihole/advanced/Scripts/database_migration/gravity" upgrade_gravityDB(){ - local database piholeDir auditFile version - database="${1}" - piholeDir="${2}" - auditFile="${piholeDir}/auditlog.list" + local database piholeDir auditFile version + database="${1}" + piholeDir="${2}" + auditFile="${piholeDir}/auditlog.list" - # Get database version - version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" + # Get database version + version="$(pihole-FTL sqlite3 -ni "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" - if [[ "$version" == "1" ]]; then - # This migration script upgrades the gravity.db file by - # adding the domain_audit table - echo -e " ${INFO} Upgrading gravity database from version 1 to 2" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql" - version=2 + if [[ "$version" == "1" ]]; then + # This migration script upgrades the gravity.db file by + # adding the domain_audit table + echo -e " ${INFO} Upgrading gravity database from version 1 to 2" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/1_to_2.sql" + version=2 - # Store audit domains in database table - if [ -e "${auditFile}" ]; then - echo -e " ${INFO} Migrating content of ${auditFile} into new database" - # database_table_from_file is defined in gravity.sh - database_table_from_file "domain_audit" "${auditFile}" - fi - fi - if [[ "$version" == "2" ]]; then - # This migration script upgrades the gravity.db file by - # renaming the regex table to regex_blacklist, and - # creating a new regex_whitelist table + corresponding linking table and views - echo -e " ${INFO} Upgrading gravity database from version 2 to 3" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql" - version=3 - fi - if [[ "$version" == "3" ]]; then - # This migration script unifies the formally separated domain - # lists into a single table with a UNIQUE domain constraint - echo -e " ${INFO} Upgrading gravity database from version 3 to 4" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql" - version=4 - fi - if [[ "$version" == "4" ]]; then - # This migration script upgrades the gravity and list views - # implementing necessary changes for per-client blocking - echo -e " ${INFO} Upgrading gravity database from version 4 to 5" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql" - version=5 - fi - if [[ "$version" == "5" ]]; then - # This migration script upgrades the adlist view - # to return an ID used in gravity.sh - echo -e " ${INFO} Upgrading gravity database from version 5 to 6" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql" - version=6 - fi - if [[ "$version" == "6" ]]; then - # This migration script adds a special group with ID 0 - # which is automatically associated to all clients not - # having their own group assignments - echo -e " ${INFO} Upgrading gravity database from version 6 to 7" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql" - version=7 - fi - if [[ "$version" == "7" ]]; then - # This migration script recreated the group table - # to ensure uniqueness on the group name - # We also add date_added and date_modified columns - echo -e " ${INFO} Upgrading gravity database from version 7 to 8" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql" - version=8 - fi - if [[ "$version" == "8" ]]; then - # This migration fixes some issues that were introduced - # in the previous migration script. - echo -e " ${INFO} Upgrading gravity database from version 8 to 9" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql" - version=9 - fi - if [[ "$version" == "9" ]]; then - # This migration drops unused tables and creates triggers to remove - # obsolete groups assignments when the linked items are deleted - echo -e " ${INFO} Upgrading gravity database from version 9 to 10" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql" - version=10 - fi - if [[ "$version" == "10" ]]; then - # This adds timestamp and an optional comment field to the client table - # These fields are only temporary and will be replaces by the columns - # defined in gravity.db.sql during gravity swapping. We add them here - # to keep the copying process generic (needs the same columns in both the - # source and the destination databases). - echo -e " ${INFO} Upgrading gravity database from version 10 to 11" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql" - version=11 - fi - if [[ "$version" == "11" ]]; then - # Rename group 0 from "Unassociated" to "Default" - echo -e " ${INFO} Upgrading gravity database from version 11 to 12" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql" - version=12 - fi - if [[ "$version" == "12" ]]; then - # Add column date_updated to adlist table - echo -e " ${INFO} Upgrading gravity database from version 12 to 13" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql" - version=13 - fi - if [[ "$version" == "13" ]]; then - # Add columns number and status to adlist table - echo -e " ${INFO} Upgrading gravity database from version 13 to 14" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql" - version=14 - fi - if [[ "$version" == "14" ]]; then - # Changes the vw_adlist created in 5_to_6 - echo -e " ${INFO} Upgrading gravity database from version 14 to 15" - pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql" - version=15 - fi + # Store audit domains in database table + if [ -e "${auditFile}" ]; then + echo -e " ${INFO} Migrating content of ${auditFile} into new database" + # database_table_from_file is defined in gravity.sh + database_table_from_file "domain_audit" "${auditFile}" + fi + fi + if [[ "$version" == "2" ]]; then + # This migration script upgrades the gravity.db file by + # renaming the regex table to regex_blacklist, and + # creating a new regex_whitelist table + corresponding linking table and views + echo -e " ${INFO} Upgrading gravity database from version 2 to 3" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/2_to_3.sql" + version=3 + fi + if [[ "$version" == "3" ]]; then + # This migration script unifies the formally separated domain + # lists into a single table with a UNIQUE domain constraint + echo -e " ${INFO} Upgrading gravity database from version 3 to 4" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/3_to_4.sql" + version=4 + fi + if [[ "$version" == "4" ]]; then + # This migration script upgrades the gravity and list views + # implementing necessary changes for per-client blocking + echo -e " ${INFO} Upgrading gravity database from version 4 to 5" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/4_to_5.sql" + version=5 + fi + if [[ "$version" == "5" ]]; then + # This migration script upgrades the adlist view + # to return an ID used in gravity.sh + echo -e " ${INFO} Upgrading gravity database from version 5 to 6" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/5_to_6.sql" + version=6 + fi + if [[ "$version" == "6" ]]; then + # This migration script adds a special group with ID 0 + # which is automatically associated to all clients not + # having their own group assignments + echo -e " ${INFO} Upgrading gravity database from version 6 to 7" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/6_to_7.sql" + version=7 + fi + if [[ "$version" == "7" ]]; then + # This migration script recreated the group table + # to ensure uniqueness on the group name + # We also add date_added and date_modified columns + echo -e " ${INFO} Upgrading gravity database from version 7 to 8" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/7_to_8.sql" + version=8 + fi + if [[ "$version" == "8" ]]; then + # This migration fixes some issues that were introduced + # in the previous migration script. + echo -e " ${INFO} Upgrading gravity database from version 8 to 9" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/8_to_9.sql" + version=9 + fi + if [[ "$version" == "9" ]]; then + # This migration drops unused tables and creates triggers to remove + # obsolete groups assignments when the linked items are deleted + echo -e " ${INFO} Upgrading gravity database from version 9 to 10" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/9_to_10.sql" + version=10 + fi + if [[ "$version" == "10" ]]; then + # This adds timestamp and an optional comment field to the client table + # These fields are only temporary and will be replaces by the columns + # defined in gravity.db.sql during gravity swapping. We add them here + # to keep the copying process generic (needs the same columns in both the + # source and the destination databases). + echo -e " ${INFO} Upgrading gravity database from version 10 to 11" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/10_to_11.sql" + version=11 + fi + if [[ "$version" == "11" ]]; then + # Rename group 0 from "Unassociated" to "Default" + echo -e " ${INFO} Upgrading gravity database from version 11 to 12" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/11_to_12.sql" + version=12 + fi + if [[ "$version" == "12" ]]; then + # Add column date_updated to adlist table + echo -e " ${INFO} Upgrading gravity database from version 12 to 13" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/12_to_13.sql" + version=13 + fi + if [[ "$version" == "13" ]]; then + # Add columns number and status to adlist table + echo -e " ${INFO} Upgrading gravity database from version 13 to 14" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/13_to_14.sql" + version=14 + fi + if [[ "$version" == "14" ]]; then + # Changes the vw_adlist created in 5_to_6 + echo -e " ${INFO} Upgrading gravity database from version 14 to 15" + pihole-FTL sqlite3 -ni "${database}" < "${scriptPath}/14_to_15.sql" + version=15 + fi } diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql index 6d57a6fe..ef445cc6 100644 --- a/advanced/Scripts/database_migration/gravity/1_to_2.sql +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -4,9 +4,9 @@ BEGIN TRANSACTION; CREATE TABLE domain_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) ); UPDATE info SET value = 2 WHERE property = 'version'; diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql index fd7c24d2..9ade340a 100644 --- a/advanced/Scripts/database_migration/gravity/2_to_3.sql +++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql @@ -8,9 +8,9 @@ ALTER TABLE regex RENAME TO regex_blacklist; CREATE TABLE regex_blacklist_by_group ( - regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (regex_blacklist_id, group_id) + regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_blacklist_id, group_id) ); INSERT INTO regex_blacklist_by_group SELECT * FROM regex_by_group; @@ -32,19 +32,19 @@ CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist CREATE TABLE regex_whitelist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT ); CREATE TABLE regex_whitelist_by_group ( - regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (regex_whitelist_id, group_id) + regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_whitelist_id, group_id) ); CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain diff --git a/advanced/Scripts/database_migration/gravity/3_to_4.sql b/advanced/Scripts/database_migration/gravity/3_to_4.sql index 05231f72..2b3d0a79 100644 --- a/advanced/Scripts/database_migration/gravity/3_to_4.sql +++ b/advanced/Scripts/database_migration/gravity/3_to_4.sql @@ -6,13 +6,13 @@ BEGIN TRANSACTION; CREATE TABLE domainlist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - type INTEGER NOT NULL DEFAULT 0, - domain TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + type INTEGER NOT NULL DEFAULT 0, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT ); ALTER TABLE whitelist ADD COLUMN type INTEGER; @@ -41,9 +41,9 @@ DROP TABLE regex_whitelist_by_group; DROP TABLE regex_blacklist_by_group; CREATE TABLE domainlist_by_group ( - domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (domainlist_id, group_id) + domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (domainlist_id, group_id) ); DROP TRIGGER tr_whitelist_update; diff --git a/advanced/Scripts/database_migration/gravity/4_to_5.sql b/advanced/Scripts/database_migration/gravity/4_to_5.sql index 4ae9f980..1436c69d 100644 --- a/advanced/Scripts/database_migration/gravity/4_to_5.sql +++ b/advanced/Scripts/database_migration/gravity/4_to_5.sql @@ -7,9 +7,9 @@ BEGIN TRANSACTION; DROP TABLE gravity; CREATE TABLE gravity ( - domain TEXT NOT NULL, - adlist_id INTEGER NOT NULL REFERENCES adlist (id), - PRIMARY KEY(domain, adlist_id) + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id), + PRIMARY KEY(domain, adlist_id) ); DROP VIEW vw_gravity; @@ -22,15 +22,15 @@ CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id CREATE TABLE client ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ip TEXT NOL NULL UNIQUE + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOL NULL UNIQUE ); CREATE TABLE client_by_group ( - client_id INTEGER NOT NULL REFERENCES client (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (client_id, group_id) + client_id INTEGER NOT NULL REFERENCES client (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (client_id, group_id) ); UPDATE info SET value = 5 WHERE property = 'version'; diff --git a/advanced/Scripts/database_migration/gravity/7_to_8.sql b/advanced/Scripts/database_migration/gravity/7_to_8.sql index ccf0c148..c6a5b35b 100644 --- a/advanced/Scripts/database_migration/gravity/7_to_8.sql +++ b/advanced/Scripts/database_migration/gravity/7_to_8.sql @@ -8,12 +8,12 @@ ALTER TABLE "group" RENAME TO "group__"; CREATE TABLE "group" ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - enabled BOOLEAN NOT NULL DEFAULT 1, - name TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - description TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + enabled BOOLEAN NOT NULL DEFAULT 1, + name TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + description TEXT ); CREATE TRIGGER tr_group_update AFTER UPDATE ON "group" diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 3f696d6d..cacf5571 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -3,90 +3,90 @@ BEGIN TRANSACTION; CREATE TABLE "group" ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - enabled BOOLEAN NOT NULL DEFAULT 1, - name TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - description TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + enabled BOOLEAN NOT NULL DEFAULT 1, + name TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + description TEXT ); INSERT INTO "group" (id,enabled,name,description) VALUES (0,1,'Default','The default group'); CREATE TABLE domainlist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - type INTEGER NOT NULL DEFAULT 0, - domain TEXT NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT, - UNIQUE(domain, type) + id INTEGER PRIMARY KEY AUTOINCREMENT, + type INTEGER NOT NULL DEFAULT 0, + domain TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT, + UNIQUE(domain, type) ); CREATE TABLE adlist ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - address TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT, - date_updated INTEGER, - number INTEGER NOT NULL DEFAULT 0, - invalid_domains INTEGER NOT NULL DEFAULT 0, - status INTEGER NOT NULL DEFAULT 0 + id INTEGER PRIMARY KEY AUTOINCREMENT, + address TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT, + date_updated INTEGER, + number INTEGER NOT NULL DEFAULT 0, + invalid_domains INTEGER NOT NULL DEFAULT 0, + status INTEGER NOT NULL DEFAULT 0 ); CREATE TABLE adlist_by_group ( - adlist_id INTEGER NOT NULL REFERENCES adlist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (adlist_id, group_id) + adlist_id INTEGER NOT NULL REFERENCES adlist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (adlist_id, group_id) ); CREATE TABLE gravity ( - domain TEXT NOT NULL, - adlist_id INTEGER NOT NULL REFERENCES adlist (id) + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id) ); CREATE TABLE info ( - property TEXT PRIMARY KEY, - value TEXT NOT NULL + property TEXT PRIMARY KEY, + value TEXT NOT NULL ); INSERT INTO "info" VALUES('version','15'); CREATE TABLE domain_audit ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) ); CREATE TABLE domainlist_by_group ( - domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (domainlist_id, group_id) + domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (domainlist_id, group_id) ); CREATE TABLE client ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ip TEXT NOT NULL UNIQUE, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOT NULL UNIQUE, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT ); CREATE TABLE client_by_group ( - client_id INTEGER NOT NULL REFERENCES client (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (client_id, group_id) + client_id INTEGER NOT NULL REFERENCES client (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (client_id, group_id) ); CREATE TRIGGER tr_adlist_update AFTER UPDATE OF address,enabled,comment ON adlist diff --git a/advanced/Templates/logrotate b/advanced/Templates/logrotate index 9a56b552..bb63b118 100644 --- a/advanced/Templates/logrotate +++ b/advanced/Templates/logrotate @@ -1,21 +1,21 @@ /var/log/pihole/pihole.log { - # su # - daily - copytruncate - rotate 5 - compress - delaycompress - notifempty - nomail + # su # + daily + copytruncate + rotate 5 + compress + delaycompress + notifempty + nomail } /var/log/pihole/FTL.log { - # su # - weekly - copytruncate - rotate 3 - compress - delaycompress - notifempty - nomail + # su # + weekly + copytruncate + rotate 3 + compress + delaycompress + notifempty + nomail } diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 29a3270d..2994b92f 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -1,79 +1,79 @@ _pihole() { - local cur prev opts opts_admin opts_checkout opts_chronometer opts_debug opts_interface opts_logging opts_privacy opts_query opts_update opts_version - COMPREPLY=() - cur="${COMP_WORDS[COMP_CWORD]}" - prev="${COMP_WORDS[COMP_CWORD-1]}" - prev2="${COMP_WORDS[COMP_CWORD-2]}" + local cur prev opts opts_admin opts_checkout opts_chronometer opts_debug opts_interface opts_logging opts_privacy opts_query opts_update opts_version + COMPREPLY=() + cur="${COMP_WORDS[COMP_CWORD]}" + prev="${COMP_WORDS[COMP_CWORD-1]}" + prev2="${COMP_WORDS[COMP_CWORD-2]}" - case "${prev}" in - "pihole") - opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" - COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) - ;; - "whitelist"|"blacklist"|"wildcard"|"regex") - opts_lists="\--delmode \--noreload \--quiet \--list \--nuke" - COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) - ;; - "admin") - opts_admin="celsius fahrenheit interface kelvin password privacylevel" - COMPREPLY=( $(compgen -W "${opts_admin}" -- ${cur}) ) - ;; - "checkout") - opts_checkout="core ftl web master dev" - COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) - ;; - "chronometer") - opts_chronometer="\--exit \--json \--refresh" - COMPREPLY=( $(compgen -W "${opts_chronometer}" -- ${cur}) ) - ;; - "debug") - opts_debug="-a" - COMPREPLY=( $(compgen -W "${opts_debug}" -- ${cur}) ) - ;; - "logging") - opts_logging="on off 'off noflush'" - COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) ) - ;; - "query") - opts_query="-adlist -all -exact" - COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) ) - ;; - "updatePihole"|"-up") - opts_update="--check-only" - COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) - ;; - "version") - opts_version="\--admin \--current \--ftl \--hash \--latest \--pihole" - COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) - ;; - "interface") - if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then - opts_interface="$(cat /proc/net/dev | cut -d: -s -f1)" - COMPREPLY=( $(compgen -W "${opts_interface}" -- ${cur}) ) - else - return 1 - fi - ;; - "privacylevel") - if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then - opts_privacy="0 1 2 3" - COMPREPLY=( $(compgen -W "${opts_privacy}" -- ${cur}) ) - else - return 1 - fi - ;; - "core"|"admin"|"ftl") - if [[ "$prev2" == "checkout" ]]; then - opts_checkout="master dev" - COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) - else - return 1 - fi - ;; - *) - return 1 - ;; - esac - return 0 + case "${prev}" in + "pihole") + opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + ;; + "whitelist"|"blacklist"|"wildcard"|"regex") + opts_lists="\--delmode \--noreload \--quiet \--list \--nuke" + COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) + ;; + "admin") + opts_admin="celsius fahrenheit interface kelvin password privacylevel" + COMPREPLY=( $(compgen -W "${opts_admin}" -- ${cur}) ) + ;; + "checkout") + opts_checkout="core ftl web master dev" + COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) + ;; + "chronometer") + opts_chronometer="\--exit \--json \--refresh" + COMPREPLY=( $(compgen -W "${opts_chronometer}" -- ${cur}) ) + ;; + "debug") + opts_debug="-a" + COMPREPLY=( $(compgen -W "${opts_debug}" -- ${cur}) ) + ;; + "logging") + opts_logging="on off 'off noflush'" + COMPREPLY=( $(compgen -W "${opts_logging}" -- ${cur}) ) + ;; + "query") + opts_query="-adlist -all -exact" + COMPREPLY=( $(compgen -W "${opts_query}" -- ${cur}) ) + ;; + "updatePihole"|"-up") + opts_update="--check-only" + COMPREPLY=( $(compgen -W "${opts_update}" -- ${cur}) ) + ;; + "version") + opts_version="\--admin \--current \--ftl \--hash \--latest \--pihole" + COMPREPLY=( $(compgen -W "${opts_version}" -- ${cur}) ) + ;; + "interface") + if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then + opts_interface="$(cat /proc/net/dev | cut -d: -s -f1)" + COMPREPLY=( $(compgen -W "${opts_interface}" -- ${cur}) ) + else + return 1 + fi + ;; + "privacylevel") + if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then + opts_privacy="0 1 2 3" + COMPREPLY=( $(compgen -W "${opts_privacy}" -- ${cur}) ) + else + return 1 + fi + ;; + "core"|"admin"|"ftl") + if [[ "$prev2" == "checkout" ]]; then + opts_checkout="master dev" + COMPREPLY=( $(compgen -W "${opts_checkout}" -- ${cur}) ) + else + return 1 + fi + ;; + *) + return 1 + ;; + esac + return 0 } complete -F _pihole pihole diff --git a/manpages/pihole.8 b/manpages/pihole.8 index fec1fa5e..52773fc0 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -184,7 +184,7 @@ Available commands and options: Specify whether the Pi-hole log should be used .br - (Logging options): + (Logging options): .br on Enable the Pi-hole log at /var/log/pihole/pihole.log .br From 2b33a0494b61eb4cbaaa26e1376914a526b7cd91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jul 2024 10:12:06 +0000 Subject: [PATCH 399/462] Bump tox from 4.15.1 to 4.16.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.15.1 to 4.16.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.15.1...4.16.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 557d6497..6d230749 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.2.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.15.1 +tox == 4.16.0 From e8802173f5eacad1671997e833bce82f3bff69ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 6 Jul 2024 20:09:24 +0000 Subject: [PATCH 400/462] Bump tox from 4.15.1 to 4.16.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.15.1 to 4.16.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.15.1...4.16.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index f68261de..73b9ad6e 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.2.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.15.1 +tox == 4.16.0 pytest-clarity == 1.0.1 From 6ffb20f09e6a24152e3fe10c2a03b07c8d401d09 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 30 Apr 2024 15:47:57 +0200 Subject: [PATCH 401/462] Add protocol validation when downloading blocklist from URL Signed-off-by: DL6ER --- gravity.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gravity.sh b/gravity.sh index 522d92f5..d1664394 100755 --- a/gravity.sh +++ b/gravity.sh @@ -622,6 +622,12 @@ gravity_DownloadBlocklistFromUrl() { fi fi + # Check for allowed protocols + if [[ $url != "http"* && $url != "https"* && $url != "file"* && $url != "ftp"* && $url != "ftps"* && $url != "sftp"* ]]; then + echo -e "${OVER} ${CROSS} ${str} Invalid protocol specified, ignoring list" + download=false + fi + if [[ "${download}" == true ]]; then # shellcheck disable=SC2086 httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${heisenbergCompensator} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null) From 5cb740ef01ef29433c20c973a2d2d84942ecfd86 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Wed, 10 Jul 2024 14:29:40 +0900 Subject: [PATCH 402/462] chore: update basic-install.sh minor fix Signed-off-by: Ikko Eltociear Ashimine --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a134c0d2..ef891ec4 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -529,7 +529,7 @@ find_IPv4_information() { local route local IPv4bare - # Find IP used to route to outside world by checking the the route to Google's public DNS server + # Find IP used to route to outside world by checking the route to Google's public DNS server route=$(ip route get 8.8.8.8) # Get just the interface IPv4 address From bdbec058e994020bc732fe284343674e94da971f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 10:14:54 +0000 Subject: [PATCH 403/462] Bump actions/setup-python from 5.1.0 to 5.1.1 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.1.0 to 5.1.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.0...v5.1.1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1da17f0a..9ce20a01 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,7 +74,7 @@ jobs: uses: actions/checkout@v4.1.7 - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.10" From e90df12053fff0a6766c7f92ffe6cfcdf14b9b21 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 13 Jul 2024 10:15:25 +0000 Subject: [PATCH 404/462] Bump actions/setup-python from 5.1.0 to 5.1.1 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.1.0 to 5.1.1. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.0...v5.1.1) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ec2b5728..6ec42e9c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -67,7 +67,7 @@ jobs: uses: actions/checkout@v4.1.7 - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.0 + uses: actions/setup-python@v5.1.1 with: python-version: "3.10" From 7ca4b59b34f4f23199c7a9c2f751df39f9bae858 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 Jul 2024 10:27:07 +0200 Subject: [PATCH 405/462] Do not test API availability outside of LoginAPI Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index c76e890e..3340bdd2 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -131,9 +131,6 @@ Main() { # https://github.com/pi-hole/FTL/pull/1715 # no need to do it here - # Test if the authentication endpoint is available - TestAPIAvailability - # Authenticate with FTL LoginAPI From 897e23089c10a6ac8c7c51dbfe5c897c7051c22c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 16 Jul 2024 10:55:28 +0200 Subject: [PATCH 406/462] Implement pihole enable/disable using the API Signed-off-by: DL6ER --- advanced/Scripts/pihole-reenable.sh | 23 ------ pihole | 123 +++++++++++++--------------- 2 files changed, 57 insertions(+), 89 deletions(-) delete mode 100755 advanced/Scripts/pihole-reenable.sh diff --git a/advanced/Scripts/pihole-reenable.sh b/advanced/Scripts/pihole-reenable.sh deleted file mode 100755 index 93ec3b95..00000000 --- a/advanced/Scripts/pihole-reenable.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# Pi-hole: A black hole for Internet advertisements -# (c) 2020 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. -# -# -# The pihole disable command has the option to set a specified time before -# blocking is automatically re-enabled. -# -# Present script is responsible for the sleep & re-enable part of the job and -# is automatically terminated if it is still running when pihole is enabled by -# other means. -# -# This ensures that pihole ends up in the correct state after a sequence of -# commands suchs as: `pihole disable 30s; pihole enable; pihole disable` - -readonly PI_HOLE_BIN_DIR="/usr/local/bin" - -sleep "${1}" -"${PI_HOLE_BIN_DIR}"/pihole enable diff --git a/pihole b/pihole index 5a3c847d..08ff5b76 100755 --- a/pihole +++ b/pihole @@ -19,9 +19,13 @@ PI_HOLE_BIN_DIR="/usr/local/bin" readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" source "${colfile}" -utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +readonly utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" source "${utilsfile}" +# Source api functions +readonly apifile="${PI_HOLE_SCRIPT_DIR}/api.sh" +source "${apifile}" + versionsfile="/etc/pihole/versions" if [ -f "${versionsfile}" ]; then # Only source versionsfile if the file exits @@ -205,73 +209,60 @@ restartDNS() { piholeEnable() { if [[ "${2}" == "-h" ]] || [[ "${2}" == "--help" ]]; then - echo "Usage: pihole disable [time] -Example: 'pihole disable', or 'pihole disable 5m' -Disable Pi-hole subsystems + echo "Usage: pihole enable/disable [time] +Example: 'pihole enable', or 'pihole disable 5m' +En- or disable Pi-hole subsystems Time: - #s Disable Pi-hole functionality for # second(s) - #m Disable Pi-hole functionality for # minute(s)" + #s En-/disable Pi-hole functionality for # second(s) + #m En-/disable Pi-hole functionality for # minute(s)" exit 0 - elif [[ "${1}" == "0" ]]; then - # Disable Pi-hole - if ! getFTLConfigValue dns.blocking.active; then - echo -e " ${INFO} Blocking already disabled, nothing to do" - exit 0 - fi - if [[ $# -gt 1 ]]; then - local error=false - if [[ "${2}" == *"s" ]]; then - tt=${2%"s"} - if [[ "${tt}" =~ ^-?[0-9]+$ ]];then - local str="Disabling blocking for ${tt} seconds" - echo -e " ${INFO} ${str}..." - local str="Blocking will be re-enabled in ${tt} seconds" - nohup "${PI_HOLE_SCRIPT_DIR}"/pihole-reenable.sh ${tt} /dev/null & - else - local error=true - fi - elif [[ "${2}" == *"m" ]]; then - tt=${2%"m"} - if [[ "${tt}" =~ ^-?[0-9]+$ ]];then - local str="Disabling blocking for ${tt} minutes" - echo -e " ${INFO} ${str}..." - local str="Blocking will be re-enabled in ${tt} minutes" - tt=$((${tt}*60)) - nohup "${PI_HOLE_SCRIPT_DIR}"/pihole-reenable.sh ${tt} /dev/null & - else - local error=true - fi - elif [[ -n "${2}" ]]; then - local error=true - else - echo -e " ${INFO} Disabling blocking" - fi - - if [[ ${error} == true ]];then - echo -e " ${COL_LIGHT_RED}Unknown format for delayed reactivation of the blocking!${COL_NC}" - echo -e " Try 'pihole disable --help' for more information." - exit 1 - fi - - local str="Pi-hole Disabled" - setFTLConfigValue dns.blocking.active false - fi - else - # Enable Pi-hole - killall -q pihole-reenable - if getFTLConfigValue dns.blocking.active; then - echo -e " ${INFO} Blocking already enabled, nothing to do" - exit 0 - fi - echo -e " ${INFO} Enabling blocking" - local str="Pi-hole Enabled" - - setFTLConfigValue dns.blocking.active true fi - restartDNS reload-lists + # Get timer + local tt="null" + if [[ $# -gt 1 ]]; then + local error=false + if [[ "${2}" == *"s" ]]; then + tt=${2%"s"} + if [[ ! "${tt}" =~ ^-?[0-9]+$ ]];then + local error=true + fi + elif [[ "${2}" == *"m" ]]; then + tt=${2%"m"} + if [[ "${tt}" =~ ^-?[0-9]+$ ]];then + tt=$((${tt}*60)) + else + local error=true + fi + elif [[ -n "${2}" ]]; then + local error=true + fi + + if [[ ${error} == true ]];then + echo -e " ${COL_LIGHT_RED}Unknown format for blocking timer!${COL_NC}" + echo -e " Try 'pihole disable --help' for more information." + exit 1 + fi + fi + + # Authenticate with the API + LoginAPI + + # Send the request + data=$(PostFTLData "dns/blocking" "{ \"blocking\": ${1}, \"timer\": ${tt} }") + + # Check the response + local extra=" forever" + local timer="$(echo "${data}"| jq --raw-output '.timer' )" + if [[ "${timer}" != "null" ]]; then + extra=" for ${timer}s" + fi + local str="Pi-hole $(echo "${data}" | jq --raw-output '.blocking')${extra}" + + # Logout from the API + LogoutAPI echo -e "${OVER} ${TICK} ${str}" } @@ -548,8 +539,8 @@ case "${1}" in "-r" | "reconfigure" ) ;; "-l" | "logging" ) ;; "uninstall" ) ;; - "enable" ) ;; - "disable" ) ;; + "enable" ) need_root=0;; + "disable" ) need_root=0;; "-d" | "debug" ) ;; "restartdns" ) ;; "-g" | "updateGravity" ) ;; @@ -591,8 +582,8 @@ case "${1}" in "-g" | "updateGravity" ) updateGravityFunc "$@";; "-l" | "logging" ) piholeLogging "$@";; "uninstall" ) uninstallFunc;; - "enable" ) piholeEnable 1;; - "disable" ) piholeEnable 0 "$2";; + "enable" ) piholeEnable true "$2";; + "disable" ) piholeEnable false "$2";; "restartdns" ) restartDNS "$2";; "reloaddns" ) restartDNS "reload";; "setpassword" ) SetWebPassword "$@";; From 3cc194594252966cd4adce76263f2b7c8c3da713 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Jul 2024 10:08:11 +0000 Subject: [PATCH 407/462] Bump pytest from 8.2.2 to 8.3.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.2.2 to 8.3.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.2.2...8.3.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 6d230749..b373944f 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.2.2 +pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 tox == 4.16.0 From 0980f9acf62f08d4e46f3c2d993b8cee88e57cd4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 27 Jul 2024 10:38:22 +0000 Subject: [PATCH 408/462] Bump pytest from 8.2.2 to 8.3.2 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.2.2 to 8.3.2. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.2.2...8.3.2) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 73b9ad6e..7c066192 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.1 -pytest == 8.2.2 +pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 tox == 4.16.0 From 2c32d485bd8d1c5892ae635e3f35065fde004e47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 27 Jul 2024 21:55:18 +0200 Subject: [PATCH 409/462] Remove obsolet getFTLPIDFile() We can get the path of the PID file via getFTLConfigValue files.pid MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/utils.sh | 19 ------------- advanced/Templates/pihole-FTL-poststop.sh | 4 +-- advanced/Templates/pihole-FTL-prestart.sh | 4 +-- advanced/Templates/pihole-FTL.service | 4 +-- pihole | 4 +-- test/test_any_utils.py | 34 +---------------------- 6 files changed, 9 insertions(+), 60 deletions(-) diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 2fe419e8..67301394 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -81,25 +81,6 @@ removeKey() { sed -i "/^${key}/d" "${file}" } -####################### -# returns path of FTL's PID file -####################### -getFTLPIDFile() { - local FTLCONFFILE="/etc/pihole/pihole-FTL.conf" - local DEFAULT_PID_FILE="/run/pihole-FTL.pid" - local FTL_PID_FILE - - if [ -s "${FTLCONFFILE}" ]; then - # if PIDFILE is not set in pihole-FTL.conf, use the default path - FTL_PID_FILE="$({ grep '^PIDFILE=' "${FTLCONFFILE}" || echo "${DEFAULT_PID_FILE}"; } | cut -d'=' -f2-)" - else - # if there is no pihole-FTL.conf, use the default path - FTL_PID_FILE="${DEFAULT_PID_FILE}" - fi - - echo "${FTL_PID_FILE}" -} - ####################### # returns FTL's PID based on the content of the pihole-FTL.pid file # diff --git a/advanced/Templates/pihole-FTL-poststop.sh b/advanced/Templates/pihole-FTL-poststop.sh index ac3898d2..b5ddbc97 100755 --- a/advanced/Templates/pihole-FTL-poststop.sh +++ b/advanced/Templates/pihole-FTL-poststop.sh @@ -1,13 +1,13 @@ #!/usr/bin/env sh -# Source utils.sh for getFTLPIDFile() +# Source utils.sh for getFTLConfigValue() PI_HOLE_SCRIPT_DIR='/opt/pihole' utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" # shellcheck disable=SC1090 . "${utilsfile}" # Get file paths -FTL_PID_FILE="$(getFTLPIDFile)" +FTL_PID_FILE="$(getFTLConfigValue files.pid)" # Cleanup rm -f /run/pihole/FTL.sock /dev/shm/FTL-* "${FTL_PID_FILE}" diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index c6817828..d807b81c 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -1,13 +1,13 @@ #!/usr/bin/env sh -# Source utils.sh for getFTLPIDFile() +# Source utils.sh for getFTLConfigValue() PI_HOLE_SCRIPT_DIR='/opt/pihole' utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" # shellcheck disable=SC1090 . "${utilsfile}" # Get file paths -FTL_PID_FILE="$(getFTLPIDFile)" +FTL_PID_FILE="$(getFTLConfigValue files.pid)" # Ensure that permissions are set so that pihole-FTL can edit all necessary files # shellcheck disable=SC2174 diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 009401fc..151d4f90 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -9,7 +9,7 @@ # Description: Enable service provided by pihole-FTL daemon ### END INIT INFO -# Source utils.sh for getFTLPIDFile(), getFTLPID() +# Source utils.sh for getFTLConfigValue(), getFTLPID() PI_HOLE_SCRIPT_DIR="/opt/pihole" utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" # shellcheck disable=SC1090 @@ -98,7 +98,7 @@ status() { trap 'cleanup; exit 1' INT HUP TERM ABRT # Get FTL's PID file path -FTL_PID_FILE="$(getFTLPIDFile)" +FTL_PID_FILE="$(getFTLConfigValue files.pid)" # Get FTL's current PID FTL_PID="$(getFTLPID "${FTL_PID_FILE}")" diff --git a/pihole b/pihole index 08ff5b76..4bb7d5e5 100755 --- a/pihole +++ b/pihole @@ -152,7 +152,7 @@ restartDNS() { svcOption="${1:-restart}" # get the current path to the pihole-FTL.pid - FTL_PID_FILE="$(getFTLPIDFile)" + FTL_PID_FILE="$(getFTLConfigValue files.pid)" # Determine if we should reload or restart if [[ "${svcOption}" =~ "reload-lists" ]]; then @@ -337,7 +337,7 @@ statusFunc() { # Determine if there is pihole-FTL service is listening local pid port ftl_pid_file block_status - ftl_pid_file="$(getFTLPIDFile)" + ftl_pid_file="$(getFTLConfigValue files.pid)" pid="$(getFTLPID ${ftl_pid_file})" diff --git a/test/test_any_utils.py b/test/test_any_utils.py index 9eee6885..59745c48 100644 --- a/test/test_any_utils.py +++ b/test/test_any_utils.py @@ -82,18 +82,6 @@ def test_key_removal_works(host): assert expected_stdout == output.stdout -def test_getFTLPIDFile_default(host): - """Confirms getFTLPIDFile returns the default PID file path""" - output = host.run( - """ - source /opt/pihole/utils.sh - getFTLPIDFile - """ - ) - expected_stdout = "/run/pihole-FTL.pid\n" - assert expected_stdout == output.stdout - - def test_getFTLPID_default(host): """Confirms getFTLPID returns the default value if FTL is not running""" output = host.run( @@ -106,27 +94,7 @@ def test_getFTLPID_default(host): assert expected_stdout == output.stdout -def test_getFTLPIDFile_and_getFTLPID_custom(host): - """Confirms getFTLPIDFile returns a custom PID file path""" - host.run( - """ - tmpfile=$(mktemp) - echo "PIDFILE=${tmpfile}" > /etc/pihole/pihole-FTL.conf - echo "1234" > ${tmpfile} - """ - ) - output = host.run( - """ - source /opt/pihole/utils.sh - FTL_PID_FILE=$(getFTLPIDFile) - getFTLPID "${FTL_PID_FILE}" - """ - ) - expected_stdout = "1234\n" - assert expected_stdout == output.stdout - - -def test_getFTLConfigValue_getFTLConfigValue(host): +def test_setFTLConfigValue_getFTLConfigValue(host): """ Confirms getFTLConfigValue works (also assumes setFTLConfigValue works) Requires FTL to be installed, so we do that first From db1431a1ae9bb796dc1a2e381f0a565c385d7c54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 27 Jul 2024 22:25:45 +0200 Subject: [PATCH 410/462] Remove obsolet files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .stickler.yml | 10 ---------- .yamllint.conf | 3 --- advanced/Templates/pihole-FTL.conf | 2 -- 3 files changed, 15 deletions(-) delete mode 100644 .stickler.yml delete mode 100644 .yamllint.conf delete mode 100644 advanced/Templates/pihole-FTL.conf diff --git a/.stickler.yml b/.stickler.yml deleted file mode 100644 index 5fdbbf1e..00000000 --- a/.stickler.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -linters: - shellcheck: - shell: bash - phpcs: - flake8: - max-line-length: 120 - yamllint: - config: ./.yamllint.conf - remarklint: diff --git a/.yamllint.conf b/.yamllint.conf deleted file mode 100644 index d1b0953b..00000000 --- a/.yamllint.conf +++ /dev/null @@ -1,3 +0,0 @@ -rules: - line-length: disable - document-start: disable diff --git a/advanced/Templates/pihole-FTL.conf b/advanced/Templates/pihole-FTL.conf deleted file mode 100644 index 269fcf9d..00000000 --- a/advanced/Templates/pihole-FTL.conf +++ /dev/null @@ -1,2 +0,0 @@ -#; Pi-hole FTL config file -#; Comments should start with #; to avoid issues with PHP and bash reading this file From 1ed5994fa575c95201df0259282b563195077e88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 28 Jul 2024 13:01:41 +0200 Subject: [PATCH 411/462] Remove temporal log file symlink code. It was introduced 2 years ago with v5.11 and always thought to be temporarily only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Templates/pihole-FTL-prestart.sh | 12 ---------- automated install/basic-install.sh | 28 ----------------------- 2 files changed, 40 deletions(-) diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index d807b81c..c604d5d4 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -32,15 +32,3 @@ find /etc/pihole -type d -exec chmod 0755 {} \; [ -f /var/log/pihole/FTL.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/FTL.log [ -f /var/log/pihole/pihole.log ] || install -m 640 -o pihole -g pihole /dev/null /var/log/pihole/pihole.log [ -f /etc/pihole/dhcp.leases ] || install -m 644 -o pihole -g pihole /dev/null /etc/pihole/dhcp.leases - - -# Backward compatibility for user-scripts that still expect log files in /var/log instead of /var/log/pihole -# Should be removed with Pi-hole v6.0 -if [ ! -f /var/log/pihole.log ]; then - ln -sf /var/log/pihole/pihole.log /var/log/pihole.log - chown -h pihole:pihole /var/log/pihole.log -fi -if [ ! -f /var/log/pihole-FTL.log ]; then - ln -sf /var/log/pihole/FTL.log /var/log/pihole-FTL.log - chown -h pihole:pihole /var/log/pihole-FTL.log -fi diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ef891ec4..ab3a2290 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2323,34 +2323,6 @@ main() { # the service before enabling causes installer to exit enable_service pihole-FTL - # If this is an update from a previous Pi-hole installation - # we need to move any existing `pihole*` logs from `/var/log` to `/var/log/pihole` - # if /var/log/pihole.log is not a symlink (set during FTL startup) move the files - # can be removed with Pi-hole v6.0 - # To be sure FTL is not running when we move the files we explicitly stop it here - - stop_service pihole-FTL &>/dev/null - - if [ ! -d /var/log/pihole/ ]; then - mkdir -m 0755 /var/log/pihole/ - fi - - # Special handling for pihole-FTL.log -> pihole/FTL.log - if [ -f /var/log/pihole-FTL.log ] && [ ! -L /var/log/pihole-FTL.log ]; then - # /var/log/pihole-FTL.log -> /var/log/pihole/FTL.log - # /var/log/pihole-FTL.log.1 -> /var/log/pihole/FTL.log.1 - # /var/log/pihole-FTL.log.2.gz -> /var/log/pihole/FTL.log.2.gz - # /var/log/pihole-FTL.log.3.gz -> /var/log/pihole/FTL.log.3.gz - # /var/log/pihole-FTL.log.4.gz -> /var/log/pihole/FTL.log.4.gz - # /var/log/pihole-FTL.log.5.gz -> /var/log/pihole/FTL.log.5.gz - for f in /var/log/pihole-FTL.log*; do mv "$f" "$(sed "s/pihole-/pihole\//" <<<"$f")"; done - fi - - # Remaining log files - if [ -f /var/log/pihole.log ] && [ ! -L /var/log/pihole.log ]; then - mv /var/log/pihole*.* /var/log/pihole/ 2>/dev/null - fi - restart_service pihole-FTL # Download and compile the aggregated block list From 58a41641ab8e499ab5a27109e04c8a1e174b3860 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Aug 2024 10:19:39 +0000 Subject: [PATCH 412/462] Bump tox from 4.16.0 to 4.17.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.16.0 to 4.17.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.16.0...4.17.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index b373944f..623e2ec5 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.1 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.16.0 +tox == 4.17.1 From 21a9410242aa337fb4213576287647cf5fd095a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Aug 2024 10:19:42 +0000 Subject: [PATCH 413/462] Bump pyyaml from 6.0.1 to 6.0.2 in /test Bumps [pyyaml](https://github.com/yaml/pyyaml) from 6.0.1 to 6.0.2. - [Release notes](https://github.com/yaml/pyyaml/releases) - [Changelog](https://github.com/yaml/pyyaml/blob/main/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/6.0.1...6.0.2) --- updated-dependencies: - dependency-name: pyyaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index b373944f..e09fcea3 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,4 +1,4 @@ -pyyaml == 6.0.1 +pyyaml == 6.0.2 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 From c42890ad5481edfd7da349b0473b22d08af44cf4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Aug 2024 10:56:14 +0000 Subject: [PATCH 414/462] Bump pyyaml from 6.0.1 to 6.0.2 in /test Bumps [pyyaml](https://github.com/yaml/pyyaml) from 6.0.1 to 6.0.2. - [Release notes](https://github.com/yaml/pyyaml/releases) - [Changelog](https://github.com/yaml/pyyaml/blob/main/CHANGES) - [Commits](https://github.com/yaml/pyyaml/compare/6.0.1...6.0.2) --- updated-dependencies: - dependency-name: pyyaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 7c066192..84887ee8 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,4 +1,4 @@ -pyyaml == 6.0.1 +pyyaml == 6.0.2 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 From 4f52c6afc056f3fe430dec91850cf79af29067bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 10 Aug 2024 12:01:35 +0000 Subject: [PATCH 415/462] Bump tox from 4.16.0 to 4.17.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.16.0 to 4.17.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.16.0...4.17.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 84887ee8..a313086b 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.16.0 +tox == 4.17.1 pytest-clarity == 1.0.1 From 27fd80c01e9430a60917bf09668e8801bfc873fb Mon Sep 17 00:00:00 2001 From: Allen <64094914+allendema@users.noreply.github.com> Date: Thu, 15 Aug 2024 02:31:33 +0200 Subject: [PATCH 416/462] [fix] [v6] typo in bash-completion allow-regex option also removes a space from double whitespace Signed-off-by: Allen Dema <64094914+allendema@users.noreply.github.com> Signed-off-by: Allen <64094914+allendema@users.noreply.github.com> --- advanced/bash-completion/pihole | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 45536d69..112e64ab 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -1,5 +1,5 @@ _pihole() { - local cur prev opts opts_checkout opts_debug opts_logging opts_query opts_update opts_version + local cur prev opts opts_checkout opts_debug opts_logging opts_query opts_update opts_version COMPREPLY=() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" @@ -10,7 +10,7 @@ _pihole() { opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard arpflush" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; - "allow"|"deny"|"wildcard"|"regex"|"allow-regx"|"allow-wild") + "allow"|"deny"|"wildcard"|"regex"|"allow-regex"|"allow-wild") opts_lists="\not \--delmode \--quiet \--list \--help" COMPREPLY=( $(compgen -W "${opts_lists}" -- ${cur}) ) ;; From 5024a98a4fdff701c118e7308d4b288f85b00701 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Aug 2024 10:06:35 +0000 Subject: [PATCH 417/462] Bump tox from 4.17.1 to 4.18.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.17.1 to 4.18.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.17.1...4.18.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 492cdf79..fe83d743 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.17.1 +tox == 4.18.0 From f6681f75fbb155d7d661a56fb12903e3841b269f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 17 Aug 2024 10:39:02 +0000 Subject: [PATCH 418/462] Bump tox from 4.17.1 to 4.18.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.17.1 to 4.18.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.17.1...4.18.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index a313086b..4bc7c372 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.17.1 +tox == 4.18.0 pytest-clarity == 1.0.1 From fc156f521ccce13449cfefeffe28f21762daa53e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 10 Aug 2024 13:15:34 +0200 Subject: [PATCH 419/462] Fix setting query logging and privacy level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index ab3a2290..5b517ab9 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -86,9 +86,9 @@ adlistFile="/etc/pihole/adlists.list" IPV4_ADDRESS=${IPV4_ADDRESS} IPV6_ADDRESS=${IPV6_ADDRESS} # Give settings their default values. These may be changed by prompts later in the script. -QUERY_LOGGING=true +QUERY_LOGGING= WEBPORT=8080 -PRIVACY_LEVEL=0 +PRIVACY_LEVEL= # Where old configs go to if a v6 migration is performed V6_CONF_MIGRATION_DIR="/etc/pihole/migration_backup_v6" @@ -2300,6 +2300,15 @@ main() { pihole -a -p "${pw}" fi + # write privacy level and logging to pihole.toml + # set on fresh installations by setPrivacyLevel() and setLogging( + if [ -n "${QUERY_LOGGING}" ]; then + pihole-FTL --config dns.queryLogging "${QUERY_LOGGING}" + fi + if [ -n "${PRIVACY_LEVEL}" ]; then + pihole-FTL --config misc.privacylevel "${PRIVACY_LEVEL}" + fi + # Migrate existing install to v6.0 migrate_dnsmasq_configs From 8e4fc27168a424a0a628cb27ed5967958ee09c0b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 24 Aug 2024 10:06:48 +0200 Subject: [PATCH 420/462] Add pihole api [endpoint] callback suitable for local API requests Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 63 ++++++++++++++++++++++++++++++++++------- pihole | 2 ++ 2 files changed, 54 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 5843c585..4776222b 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -146,19 +146,22 @@ GetFTLData() { response=$(curl -skS -w "%{http_code}" -X GET "${API_URL}$1" -H "Accept: application/json" -H "sid: ${SID}" ) # status are the last 3 characters - status=$(printf %s "${response#"${response%???}"}") + status="${response#"${response%???}"}" # data is everything from response without the last 3 characters - data=$(printf %s "${response%???}") + data="${response%???}" - if [ "${status}" = 200 ]; then - # response OK - printf %s "${data}" - elif [ "${status}" = 000 ]; then - # connection lost - echo "000" - elif [ "${status}" = 401 ]; then - # unauthorized - echo "401" + if [ "${2}" = "raw" ]; then + # return the raw response + echo "${response}" + else + # return only the data + if [ "${status}" = 200 ]; then + # response OK + echo "${data}" + else + # connection lost + echo "${status}" + fi fi } @@ -226,3 +229,41 @@ secretRead() { # restore original terminal settings stty "${stty_orig}" } + +apiFunc() { + local data response status status_col + + # Authenticate with the API + LoginAPI + + echo "Requesting: ${COL_PURPLE}GET ${COL_CYAN}${API_URL}${COL_YELLOW}$1${COL_NC}" + echo "" + + # Get the data from the API + response=$(GetFTLData "$1" raw) + + # status are the last 3 characters + status="${response#"${response%???}"}" + # data is everything from response without the last 3 characters + data="${response%???}" + + # Output the status (200 -> green, else red) + if [ "${status}" = 200 ]; then + status_col="${COL_GREEN}" + else + status_col="${COL_RED}" + fi + echo "Status: ${status_col}${status}${COL_NC}" + + # Output the data. Format it with jq if available and data is actually JSON. + # Otherwise just print it + echo "Data:" + if command -v jq >/dev/null && echo "${data}" | jq . >/dev/null 2>&1; then + echo "${data}" | jq . + else + echo "${data}" + fi + + # Delete the session + LogoutAPI +} diff --git a/pihole b/pihole index 4bb7d5e5..8a9aa297 100755 --- a/pihole +++ b/pihole @@ -550,6 +550,7 @@ case "${1}" in "updatechecker" ) ;; "arpflush" ) ;; "-t" | "tail" ) ;; + "api" ) need_root=0;; * ) helpFunc;; esac @@ -591,5 +592,6 @@ case "${1}" in "updatechecker" ) shift; updateCheckFunc "$@";; "arpflush" ) arpFunc "$@";; "-t" | "tail" ) tailFunc "$2";; + "api" ) apiFunc "$2";; * ) helpFunc;; esac From b304562c8ebbb9ecf3a678bd06be2ffe72e90161 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 24 Aug 2024 10:37:58 +0200 Subject: [PATCH 421/462] Add session details to API command output Signed-off-by: DL6ER --- advanced/Scripts/api.sh | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 4776222b..43c91d69 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -88,6 +88,9 @@ LoginAPI() { # Exit early if authentication is not needed if [ "${needAuth}" = false ]; then + if [ "${1}" = "verbose" ]; then + echo "API Authentication: Not needed" + fi return fi @@ -95,8 +98,15 @@ LoginAPI() { if [ -r /etc/pihole/cli_pw ]; then password=$(cat /etc/pihole/cli_pw) + if [ "${1}" = "verbose" ]; then + echo "API Authentication: Trying to use CLI password" + fi + # Try to authenticate using the CLI password - Authentication + Authentication "${1}" + + elif [ "${1}" = "verbose" ]; then + echo "API Authentication: CLI password not available" fi @@ -109,7 +119,7 @@ LoginAPI() { secretRead; printf '\n' # Try to authenticate again - Authentication + Authentication "${1}" done } @@ -124,6 +134,14 @@ Authentication() { # obtain validity and session ID from session response validSession=$(echo "${sessionResponse}"| jq .session.valid 2>/dev/null) SID=$(echo "${sessionResponse}"| jq --raw-output .session.sid 2>/dev/null) + + if [ "${1}" = "verbose" ]; then + if [ "${validSession}" = true ]; then + echo "API Authentication: ${COL_GREEN}Success${COL_NC}" + else + echo "API Authentication: ${COL_RED}Failed${COL_NC}" + fi + fi } LogoutAPI() { @@ -134,10 +152,12 @@ LogoutAPI() { deleteResponse=$(curl -skS -o /dev/null -w "%{http_code}" -X DELETE "${API_URL}auth" -H "Accept: application/json" -H "sid: ${SID}") case "${deleteResponse}" in - "401") printf "%b" "Logout attempt without a valid session. Unauthorized!\n";; - esac; + "401") echo "Logout attempt without a valid session. Unauthorized!";; + "204") if [ "${1}" = "verbose" ]; then echo "API Logout: ${COL_GREEN}Success${COL_NC} (session deleted)"; fi;; + esac; + elif [ "${1}" = "verbose" ]; then + echo "API Logout: ${COL_GREEN}Success${COL_NC} (no valid session)" fi - } GetFTLData() { @@ -234,7 +254,8 @@ apiFunc() { local data response status status_col # Authenticate with the API - LoginAPI + LoginAPI verbose + echo "" echo "Requesting: ${COL_PURPLE}GET ${COL_CYAN}${API_URL}${COL_YELLOW}$1${COL_NC}" echo "" @@ -265,5 +286,5 @@ apiFunc() { fi # Delete the session - LogoutAPI + LogoutAPI verbose } From 72afa1073da05a0e7ddb22c0acdb0b63190b8284 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sun, 25 Aug 2024 21:47:07 +0100 Subject: [PATCH 422/462] Add missing creation of table antigravity in migration script 16 to 17 Signed-off-by: Adam Warner --- advanced/Scripts/database_migration/gravity/16_to_17.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity/16_to_17.sql b/advanced/Scripts/database_migration/gravity/16_to_17.sql index 23532e3a..ed2ec7c5 100644 --- a/advanced/Scripts/database_migration/gravity/16_to_17.sql +++ b/advanced/Scripts/database_migration/gravity/16_to_17.sql @@ -8,6 +8,12 @@ ALTER TABLE adlist ADD COLUMN type INTEGER NOT NULL DEFAULT 0; UPDATE adlist SET type = 0; +CREATE TABLE antigravity +( + domain TEXT NOT NULL, + adlist_id INTEGER NOT NULL REFERENCES adlist (id) +); + CREATE VIEW vw_antigravity AS SELECT domain, adlist_by_group.group_id AS group_id FROM antigravity LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = antigravity.adlist_id From a302d7b5d741f2ff27de6ed8be2149bca3c6ba07 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 22 Aug 2024 21:22:52 +0200 Subject: [PATCH 423/462] Make the help text of "pihole checkout [what] [branch]" more colorful Signed-off-by: DL6ER --- advanced/Scripts/piholeCheckout.sh | 11 ++++++----- automated install/basic-install.sh | 7 ++++--- pihole | 24 +++++++++++++----------- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 41fd8606..c7780cd0 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -77,7 +77,7 @@ checkout() { if [[ "${1}" == "dev" ]] ; then # Shortcut to check out development branches - echo -e " ${INFO} Shortcut \"dev\" detected - checking out development / devel branches..." + echo -e " ${INFO} Shortcut \"${COL_YELLOW}dev${COL_NC}\" detected - checking out development / devel branches..." echo "" echo -e " ${INFO} Pi-hole Core" fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "development" || { echo " ${CROSS} Unable to pull Core development branch"; exit 1; } @@ -92,7 +92,7 @@ checkout() { chmod 644 /etc/pihole/ftlbranch elif [[ "${1}" == "master" ]] ; then # Shortcut to check out master branches - echo -e " ${INFO} Shortcut \"master\" detected - checking out master branches..." + echo -e " ${INFO} Shortcut \"${COL_YELLOW}master${COL_NC}\" detected - checking out master branches..." echo -e " ${INFO} Pi-hole core" fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "master" || { echo " ${CROSS} Unable to pull Core master branch"; exit 1; } echo -e " ${INFO} Web interface" @@ -123,7 +123,7 @@ checkout() { echo "" # Have the user choose the branch they want if ! (for e in "${corebranches[@]}"; do [[ "$e" == "${2}" ]] && exit 0; done); then - echo -e " ${INFO} Requested branch \"${2}\" is not available" + echo -e " ${INFO} Requested branch \"${COL_CYAN}${2}${COL_NC}\" is not available" echo -e " ${INFO} Available branches for Core are:" for e in "${corebranches[@]}"; do echo " - $e"; done exit 1 @@ -150,7 +150,7 @@ checkout() { echo "" # Have the user choose the branch they want if ! (for e in "${webbranches[@]}"; do [[ "$e" == "${2}" ]] && exit 0; done); then - echo -e " ${INFO} Requested branch \"${2}\" is not available" + echo -e " ${INFO} Requested branch \"${COL_CYAN}${2}${COL_NC}\" is not available" echo -e " ${INFO} Available branches for Web Admin are:" for e in "${webbranches[@]}"; do echo " - $e"; done exit 1 @@ -164,6 +164,7 @@ checkout() { path="${2}/${binary}" oldbranch="$(pihole-FTL -b)" + echo -e " ${INFO} Checking for branch ${COL_CYAN}${2}${COL_NC} on https://ftl.pi-hole.net" check_download_exists "$path" local ret=$? if [ $ret -eq 0 ]; then @@ -193,7 +194,7 @@ checkout() { fi else - echo -e " ${INFO} Requested option \"${1}\" is not available" + echo -e " ${CROSS} Requested option \"${1}\" is not available" exit 1 fi diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 5b517ab9..1b11f2f2 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -16,9 +16,10 @@ # # curl -sSL https://install.pi-hole.net | bash -# -e option instructs bash to immediately exit if any command [1] has a non-zero exit status -# We do not want users to end up with a partially working install, so we exit the script -# instead of continuing the installation with something broken +# -e option instructs bash to immediately exit if any command [1] has a non-zero +# exit status We do not want users to end up with a partially working install, +# so we exit the script instead of continuing the installation with something +# broken set -e # Append common folders to the PATH to ensure that all basic commands are available. diff --git a/pihole b/pihole index 4bb7d5e5..93545cb7 100755 --- a/pihole +++ b/pihole @@ -408,19 +408,21 @@ piholeCheckoutFunc() { unsupportedFunc else if [[ "$2" == "-h" ]] || [[ "$2" == "--help" ]]; then - echo "Usage: pihole checkout [repo] [branch] - Example: 'pihole checkout master' or 'pihole checkout core dev' - Switch Pi-hole subsystems to a different GitHub branch + echo "Switch Pi-hole subsystems to a different GitHub branch + Usage: ${COL_GREEN}pihole checkout${COL_NC} ${COL_YELLOW}shortcut${COL_NC} + or ${COL_GREEN}pihole checkout${COL_NC} ${COL_PURPLE}repo${COL_NC} ${COL_CYAN}branch${COL_NC} + Example: ${COL_GREEN}pihole checkout${COL_NC} ${COL_YELLOW}master${COL_NC} + or ${COL_GREEN}pihole checkout${COL_NC} ${COL_PURPLE}ftl ${COL_CYAN}development${COL_NC} - Repositories: - core [branch] Change the branch of Pi-hole's core subsystem - web [branch] Change the branch of Web Interface subsystem - ftl [branch] Change the branch of Pi-hole's FTL subsystem + Shortcuts: + ${COL_YELLOW}master${COL_NC} Update all subsystems to the latest stable release + ${COL_YELLOW}dev${COL_NC} Update all subsystems to the latest development release + + Individual components: + ${COL_PURPLE}core${COL_NC} ${COL_CYAN}branch${COL_NC} Change the branch of Pi-hole's core subsystem + ${COL_PURPLE}web${COL_NC} ${COL_CYAN}branch${COL_NC} Change the branch of the web interface subsystem + ${COL_PURPLE}ftl${COL_NC} ${COL_CYAN}branch${COL_NC} Change the branch of Pi-hole's FTL subsystem" - Branches: - master Update subsystems to the latest stable release - dev Update subsystems to the latest development release - branchname Update subsystems to the specified branchname" exit 0 fi From 9e9c985245308a2ee36a69bbbabb0b2bdf21441f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 26 Aug 2024 06:11:42 +0200 Subject: [PATCH 424/462] FTL checkout: Check for availability of branches before trying to download from the webserver. Also, fix check_download_exists() possibly killing the script on non-availability of requested branches Signed-off-by: DL6ER --- advanced/Scripts/piholeCheckout.sh | 44 +++++++++++++++++++++--------- automated install/basic-install.sh | 17 +++++------- pihole | 1 + 3 files changed, 39 insertions(+), 23 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index c7780cd0..65e6ebb8 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -161,34 +161,52 @@ checkout() { elif [[ "${1}" == "ftl" ]] ; then local path local oldbranch + local existing=false path="${2}/${binary}" oldbranch="$(pihole-FTL -b)" - echo -e " ${INFO} Checking for branch ${COL_CYAN}${2}${COL_NC} on https://ftl.pi-hole.net" - check_download_exists "$path" - local ret=$? - if [ $ret -eq 0 ]; then - echo " ${TICK} Branch ${2} exists" + # Check if requested branch is available + echo -e " ${INFO} Checking for availability of branch ${COL_CYAN}${2}${COL_NC} on GitHub" + ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep "refs/heads" | cut -d'/' -f3- -) ) + # If returned array is empty -> connectivity issue + if [[ ${#ftlbranches[@]} -eq 0 ]]; then + echo -e " ${CROSS} Unable to fetch branches from GitHub. Please check your Internet connection and try again later." + exit 1 + fi + + for e in "${ftlbranches[@]}"; do [[ "$e" == "${2}" ]] && existing=true; done + if [[ "${existing}" == false ]]; then + echo -e " ${CROSS} Requested branch is not available\n" + echo -e " ${INFO} Available branches are:" + for e in "${ftlbranches[@]}"; do echo " - $e"; done + exit 1 + fi + echo -e " ${TICK} Branch ${2} exists on GitHub" + + echo -e " ${INFO} Checking for ${COL_YELLOW}${binary}${COL_NC} binary on https://ftl.pi-hole.net" + + if check_download_exists "$path"; then + echo " ${TICK} Binary exists" echo "${2}" > /etc/pihole/ftlbranch chmod 644 /etc/pihole/ftlbranch - echo -e " ${INFO} Switching to branch: \"${2}\" from \"${oldbranch}\"" + echo -e " ${INFO} Switching to branch: ${COL_CYAN}${2}${COL_NC} from ${COL_CYAN}${oldbranch}${COL_NC}" FTLinstall "${binary}" restart_service pihole-FTL enable_service pihole-FTL # Update local and remote versions via updatechecker /opt/pihole/updatecheck.sh else - if [[ $ret -eq 1 ]]; then - echo " ${CROSS} Requested branch \"${2}\" is not available" - ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep 'heads' | sed 's/refs\/heads\///;s/ //g' | awk '{print $2}') ) - echo -e " ${INFO} Available branches for FTL are:" - for e in "${ftlbranches[@]}"; do echo " - $e"; done + if [ $? -eq 1 ]; then + # Binary for requested branch is not available, may still be + # int he process of being built or CI build job failed + printf " %b Binary for requested branch is not available, please try again later.\\n" ${CROSS} + printf " If the issue persists, please contact Pi-hole Support and ask them to re-generate the binary.\\n" exit 1 - elif [[ $ret -eq 2 ]]; then + elif [ $? -eq 2 ]; then printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" exit 1 else - printf " %b Unknown error. Please contact Pi-hole Support\\n" "${CROSS}" + printf " %b Unknown checkout error. Please contact Pi-hole Support\\n" "${CROSS}" exit 1 fi fi diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1b11f2f2..9927b374 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -16,10 +16,9 @@ # # curl -sSL https://install.pi-hole.net | bash -# -e option instructs bash to immediately exit if any command [1] has a non-zero -# exit status We do not want users to end up with a partially working install, -# so we exit the script instead of continuing the installation with something -# broken +# -e option instructs bash to immediately exit if any command [1] has a non-zero exit status +# We do not want users to end up with a partially working install, so we exit the script +# instead of continuing the installation with something broken set -e # Append common folders to the PATH to ensure that all basic commands are available. @@ -1694,7 +1693,7 @@ update_dialogs() { check_download_exists() { # Check if the download exists and we can reach the server - status=$(curl --head --silent "https://ftl.pi-hole.net/${1}" | head -n 1) + local status=$(curl --head --silent "https://ftl.pi-hole.net/${1}" | head -n 1) # Check the status code if grep -q "200" <<<"$status"; then @@ -2012,14 +2011,12 @@ FTLcheckUpdate() { # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! # shellcheck disable=SC1090 - check_download_exists "$path" - local ret=$? - if [ $ret -ne 0 ]; then - if [[ $ret -eq 1 ]]; then + if ! check_download_exists "$path"; then + if [ $? -eq 1 ]; then printf " %b Branch \"%s\" is not available.\\n" "${INFO}" "${ftlBranch}" printf " %b Use %bpihole checkout ftl [branchname]%b to switch to a valid branch.\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}" return 2 - elif [[ $ret -eq 2 ]]; then + elif [ $? -eq 2 ]; then printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" return 3 else diff --git a/pihole b/pihole index 93545cb7..f4951d27 100755 --- a/pihole +++ b/pihole @@ -411,6 +411,7 @@ piholeCheckoutFunc() { echo "Switch Pi-hole subsystems to a different GitHub branch Usage: ${COL_GREEN}pihole checkout${COL_NC} ${COL_YELLOW}shortcut${COL_NC} or ${COL_GREEN}pihole checkout${COL_NC} ${COL_PURPLE}repo${COL_NC} ${COL_CYAN}branch${COL_NC} + Example: ${COL_GREEN}pihole checkout${COL_NC} ${COL_YELLOW}master${COL_NC} or ${COL_GREEN}pihole checkout${COL_NC} ${COL_PURPLE}ftl ${COL_CYAN}development${COL_NC} From e8582f774099928741351fcc961f407977c663e6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 26 Aug 2024 19:44:42 +0200 Subject: [PATCH 425/462] Add pihole -h and autocompletion entry Signed-off-by: DL6ER --- advanced/bash-completion/pihole | 2 +- pihole | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 112e64ab..674eeea4 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -7,7 +7,7 @@ _pihole() { case "${prev}" in "pihole") - opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard arpflush" + opts="allow allow-regex allow-wild deny checkout debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard arpflush api" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; "allow"|"deny"|"wildcard"|"regex"|"allow-regex"|"allow-wild") diff --git a/pihole b/pihole index 8a9aa297..ed4dd67a 100755 --- a/pihole +++ b/pihole @@ -484,6 +484,7 @@ Debugging Options: -t, tail [arg] View the live output of the Pi-hole log. Add an optional argument to filter the log (regular expressions are supported) + api Query the Pi-hole API at Options: From cfbf3f61cda1b89d27db42e96b1a122018607fbd Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 26 Aug 2024 21:30:12 +0100 Subject: [PATCH 426/462] Update advanced/Scripts/database_migration/gravity/16_to_17.sql Co-authored-by: Dominik Signed-off-by: Adam Warner --- advanced/Scripts/database_migration/gravity/16_to_17.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/database_migration/gravity/16_to_17.sql b/advanced/Scripts/database_migration/gravity/16_to_17.sql index ed2ec7c5..47b631cc 100644 --- a/advanced/Scripts/database_migration/gravity/16_to_17.sql +++ b/advanced/Scripts/database_migration/gravity/16_to_17.sql @@ -8,7 +8,7 @@ ALTER TABLE adlist ADD COLUMN type INTEGER NOT NULL DEFAULT 0; UPDATE adlist SET type = 0; -CREATE TABLE antigravity +CREATE TABLE IF NOT EXISTS antigravity ( domain TEXT NOT NULL, adlist_id INTEGER NOT NULL REFERENCES adlist (id) From a21b4c5203f1ff0fba37a34b2716c86c71152d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 27 Aug 2024 21:24:50 +0200 Subject: [PATCH 427/462] Update existing logrotate files to inlcude webserver.log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 35 +++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9927b374..6b65313e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1496,6 +1496,7 @@ create_pihole_user() { installLogrotate() { local str="Installing latest logrotate script" local target=/etc/pihole/logrotate + local logfileUpdate=false printf "\\n %b %s..." "${INFO}" "${str}" if [[ -f ${target} ]]; then @@ -1506,17 +1507,35 @@ installLogrotate() { sed -i 's/\/var\/log\/pihole-FTL.log/\/var\/log\/pihole\/FTL.log/g' ${target} printf "\\n\\t%b Old log file paths updated in existing logrotate file. \\n" "${INFO}" - return 3 + logfileUpdate=true fi - printf "\\n\\t%b Existing logrotate file found. No changes made.\\n" "${INFO}" - # Return value isn't that important, using 2 to indicate that it's not a fatal error but - # the function did not complete. - return 2 + # Account for added webserver.log in v6.0 + if ! grep -q "/var/log/pihole/webserver.log" ${target}; then + echo "/var/log/pihole/webserver.log { +# su # +weekly +copytruncate +rotate 3 +compress +delaycompress +notifempty +nomail +}" >> ${target} + + printf "\\n\\t%b webserver.log added to logrotate file. \\n" "${INFO}" + logfileUpdate=true + fi + if [[ "${logfileUpdate}" == false ]]; then + printf "\\n\\t%b Existing logrotate file found. No changes made.\\n" "${INFO}" + return + fi + else + # Copy the file over from the local repo + # Logrotate config file must be owned by root and not writable by group or other + install -o root -g root -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} fi - # Copy the file over from the local repo - # Logrotate config file must be owned by root and not writable by group or other - install -o root -g root -D -m 644 -T "${PI_HOLE_LOCAL_REPO}"/advanced/Templates/logrotate ${target} + # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. From 5e69078ed10beaf4067fb2b10749b71b833a4d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 1 Sep 2024 19:36:29 +0200 Subject: [PATCH 428/462] Disable SELINUX on CentOS 9 test dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/_centos_9.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/_centos_9.Dockerfile b/test/_centos_9.Dockerfile index 2e3d055a..7e3c5b3a 100644 --- a/test/_centos_9.Dockerfile +++ b/test/_centos_9.Dockerfile @@ -1,4 +1,6 @@ FROM quay.io/centos/centos:stream9 +# Disable SELinux +RUN echo "SELINUX=disabled" > /etc/selinux/config RUN yum install -y --allowerasing curl git initscripts ENV GITDIR=/etc/.pihole From 632d8af4b069ce4f965701bab96766e65b3850fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 1 Sep 2024 19:36:29 +0200 Subject: [PATCH 429/462] Disable SELINUX on CentOS 9 test dockerfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/_centos_9.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/_centos_9.Dockerfile b/test/_centos_9.Dockerfile index 6ccd18b6..35f7b3f8 100644 --- a/test/_centos_9.Dockerfile +++ b/test/_centos_9.Dockerfile @@ -1,4 +1,6 @@ FROM quay.io/centos/centos:stream9 +# Disable SELinux +RUN echo "SELINUX=disabled" > /etc/selinux/config RUN yum install -y --allowerasing curl git initscripts ENV GITDIR /etc/.pihole From b4444023a2f9620b3f7a2a7971f0a8d3fd5e54d7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 25 Mar 2024 05:49:24 +0100 Subject: [PATCH 430/462] Wait after restarting FTL before trying to check version Signed-off-by: DL6ER --- advanced/Scripts/piholeCheckout.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 65e6ebb8..0514b8eb 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -193,6 +193,19 @@ checkout() { FTLinstall "${binary}" restart_service pihole-FTL enable_service pihole-FTL + str="Restarting FTL..." + echo -ne " ${INFO} ${str}" + # Wait until name resolution is working again after restarting FTL, + # so that the updatechecker can run successfully and does not fail + # trying to resolve github.com + until getent hosts github.com &> /dev/null; do + # Append one dot for each second waiting + str="${str}." + echo -ne " ${OVER} ${INFO} ${str}" + sleep 1 + done + echo -e " ${OVER} ${TICK} Restarted FTL service" + # Update local and remote versions via updatechecker /opt/pihole/updatecheck.sh else From bcef4f0c97637f7d17a94fa8130dda7785ba6264 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 2 Sep 2024 18:59:42 +0200 Subject: [PATCH 431/462] pihole status should return (= exit) early on error instead of continuing the script Signed-off-by: DL6ER --- pihole | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pihole b/pihole index f3e0a73a..98f478a3 100755 --- a/pihole +++ b/pihole @@ -346,7 +346,7 @@ statusFunc() { "web") echo "-1";; *) echo -e " ${CROSS} DNS service is NOT running";; esac - return 0 + exit 0 else # get the DNS port pihole-FTL is listening on port="$(getFTLConfigValue dns.port)" @@ -355,7 +355,7 @@ statusFunc() { "web") echo "-1";; *) echo -e " ${CROSS} DNS service is NOT listening";; esac - return 0 + exit 0 else if [[ "${1}" != "web" ]]; then echo -e " ${TICK} FTL is listening on port ${port}" @@ -377,7 +377,8 @@ statusFunc() { *) echo -e " ${CROSS} Pi-hole blocking is disabled";; esac fi -exit 0 + + exit 0 } tailFunc() { From f66707bd3e65a625e8fa7195dc5e47851c48febd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Sep 2024 20:30:11 +0000 Subject: [PATCH 432/462] Bump actions/setup-python from 5.1.1 to 5.2.0 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5.1.1 to 5.2.0. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5.1.1...v5.2.0) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9ce20a01..cbade0e5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,7 +74,7 @@ jobs: uses: actions/checkout@v4.1.7 - name: Set up Python 3.10 - uses: actions/setup-python@v5.1.1 + uses: actions/setup-python@v5.2.0 with: python-version: "3.10" From 76d78632700c84f63d4a34e7336ec7c9a8b2c71f Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 2 Sep 2024 22:22:24 +0100 Subject: [PATCH 433/462] remove development-v6 references from dependabot yml Signed-off-by: Adam Warner --- .github/dependabot.yml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 20163f5e..e140f792 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -20,24 +20,3 @@ updates: target-branch: development reviewers: - "pi-hole/core-maintainers" -# As above, but for development-v6 -- package-ecosystem: github-actions - directory: "/" - schedule: - interval: weekly - day: saturday - time: "10:00" - open-pull-requests-limit: 10 - target-branch: development-v6 - reviewers: - - "pi-hole/core-maintainers" -- package-ecosystem: pip - directory: "/test" - schedule: - interval: weekly - day: saturday - time: "10:00" - open-pull-requests-limit: 10 - target-branch: development-v6 - reviewers: - - "pi-hole/core-maintainers" From cd7226d5e98114dc7f506700cc7c4c7d38a43f93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 26 Jul 2024 13:19:11 +0200 Subject: [PATCH 434/462] Remove obsolet Debian 10 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- .github/workflows/test.yml | 1 - test/_debian_10.Dockerfile | 17 ----------------- test/tox.debian_10.ini | 10 ---------- 3 files changed, 28 deletions(-) delete mode 100644 test/_debian_10.Dockerfile delete mode 100644 test/tox.debian_10.ini diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index cbade0e5..0eef25f0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -56,7 +56,6 @@ jobs: matrix: distro: [ - debian_10, debian_11, debian_12, ubuntu_20, diff --git a/test/_debian_10.Dockerfile b/test/_debian_10.Dockerfile deleted file mode 100644 index dc813ac2..00000000 --- a/test/_debian_10.Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM buildpack-deps:buster-scm - -ENV GITDIR=/etc/.pihole -ENV SCRIPTDIR=/opt/pihole - -RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole -ADD . $GITDIR -RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $SCRIPTDIR/ -ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR - -RUN true && \ - chmod +x $SCRIPTDIR/* - -ENV SKIP_INSTALL=true -ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net - -#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/tox.debian_10.ini b/test/tox.debian_10.ini deleted file mode 100644 index 9995a852..00000000 --- a/test/tox.debian_10.ini +++ /dev/null @@ -1,10 +0,0 @@ -[tox] -envlist = py3 - -[testenv:py3] -allowlist_externals = docker -deps = -rrequirements.txt -setenv = - COLUMNS=120 -commands = docker buildx build --load --progress plain -f _debian_10.Dockerfile -t pytest_pihole:test_container ../ - pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py From 50dcd8d62fbcc304940717cd0c99924e68bad509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 3 Sep 2024 22:31:13 +0200 Subject: [PATCH 435/462] Add fallback option for OS check without hard-coded nameserver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 33 ++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 6b65313e..873d6d7c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -177,7 +177,7 @@ os_check() { detected_os=$(grep '^ID=' /etc/os-release | cut -d '=' -f2 | tr -d '"') detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') - # Test via IPv4 + # Test via IPv4 and hardcoded nameserver ns1.pi-hole.net cmdResult="$( dig -4 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1 echo $? @@ -198,7 +198,7 @@ os_check() { fi fi - # Try again via IPv6 + # Try again via IPv6 and hardcoded nameserver ns1.pi-hole.net if [ "$valid_response" = false ]; then unset valid_response @@ -223,6 +223,31 @@ os_check() { fi fi + # Try again without hardcoded nameserver + if [ "$valid_response" = false ]; then + unset valid_response + + cmdResult="$( + dig +short -t txt "${remote_os_domain}" 2>&1 + echo $? + )" + # Gets the return code of the previous command (last line) + digReturnCode="${cmdResult##*$'\n'}" + + if [ ! "${digReturnCode}" == "0" ]; then + valid_response=false + else + # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid + response="${cmdResult%%$'\n'*}" + # If the value of ${response} is a single 0, then this is the return code, not an actual response. + if [ "${response}" == 0 ]; then + valid_response=false + else + valid_response=true + fi + fi + fi + if [ "$valid_response" = true ]; then IFS=" " read -r -a supportedOS < <(echo "${response}" | tr -d '"') for distro_and_versions in "${supportedOS[@]}"; do @@ -259,8 +284,8 @@ os_check() { printf " %b %bRetrieval of supported OS list failed. %s. %b\\n" "${CROSS}" "${COL_LIGHT_RED}" "${errStr}" "${COL_NC}" printf " %bUnable to determine if the detected OS (%s %s) is supported%b\\n" "${COL_LIGHT_RED}" "${detected_os^}" "${detected_version}" "${COL_NC}" printf " Possible causes for this include:\\n" - printf " - Firewall blocking certain DNS lookups from Pi-hole device\\n" - printf " - ns1.pi-hole.net being blocked (required to obtain TXT record from versions.pi-hole.net containing supported operating systems)\\n" + printf " - Firewall blocking DNS lookups from Pi-hole device to ns1.pi-hole.net\\n" + printf " - DNS resolution issues of the host system\\n" printf " - Other internet connectivity issues\\n" else printf " %b %bUnsupported OS detected: %s %s%b\\n" "${CROSS}" "${COL_LIGHT_RED}" "${detected_os^}" "${detected_version}" "${COL_NC}" From a57d539098aefc390900162f3a5e891dd91eddc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 3 Sep 2024 23:02:15 +0200 Subject: [PATCH 436/462] Remove leftover DNS check via dig as it now only doubles what we already have with getent hosts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/gravity.sh b/gravity.sh index d1664394..6a8918db 100755 --- a/gravity.sh +++ b/gravity.sh @@ -308,18 +308,6 @@ gravity_CheckDNSResolutionAvailable() { exit 1 fi - # If the /etc/resolv.conf contains resolvers other than 127.0.0.1 then the local dnsmasq will not be queried and pi.hole is NXDOMAIN. - # This means that even though name resolution is working, the getent hosts check fails and the holddown timer keeps ticking and eventually fails - # So we check the output of the last command and if it failed, attempt to use dig +short as a fallback - if timeout 4 dig +short "${lookupDomain}" &>/dev/null; then - if [[ -n "${secs:-}" ]]; then - echo -e "${OVER} ${TICK} DNS resolution is now available\\n" - fi - return 0 - elif [[ -n "${secs:-}" ]]; then - echo -e "${OVER} ${CROSS} DNS resolution is not available" - exit 1 - fi # Determine error output message if pgrep pihole-FTL &>/dev/null; then From ba2d10c65eca2fcf205b9d82a2d1901df5f946e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 3 Sep 2024 23:03:59 +0200 Subject: [PATCH 437/462] DNS check in gravity should not check for FTL as it might not be the host's name server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 8 -------- 1 file changed, 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 6a8918db..7ca695f5 100755 --- a/gravity.sh +++ b/gravity.sh @@ -309,14 +309,6 @@ gravity_CheckDNSResolutionAvailable() { fi - # Determine error output message - if pgrep pihole-FTL &>/dev/null; then - echo -e " ${CROSS} DNS resolution is currently unavailable" - else - echo -e " ${CROSS} DNS service is not running" - "${PIHOLE_COMMAND}" restartdns - fi - # Ensure DNS server is given time to be resolvable secs="120" echo -ne " ${INFO} Time until retry: ${secs}" From f80efa51aac22611ff5d0fa031c80cd7b1287e39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 3 Sep 2024 23:20:00 +0200 Subject: [PATCH 438/462] Wait until DNS resolution is available MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 27 ++++++++++----------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/gravity.sh b/gravity.sh index 7ca695f5..1ac0e282 100755 --- a/gravity.sh +++ b/gravity.sh @@ -298,29 +298,22 @@ gravity_CheckDNSResolutionAvailable() { # Determine if $lookupDomain is resolvable if timeout 4 getent hosts "${lookupDomain}" &>/dev/null; then - # Print confirmation of resolvability if it had previously failed - if [[ -n "${secs:-}" ]]; then - echo -e "${OVER} ${TICK} DNS resolution is now available\\n" - fi + echo -e "${OVER} ${TICK} DNS resolution is available\\n" return 0 - elif [[ -n "${secs:-}" ]]; then - echo -e "${OVER} ${CROSS} DNS resolution is not available" - exit 1 + else + echo -e " ${CROSS} DNS resolution is currently unavailable" fi - - # Ensure DNS server is given time to be resolvable - secs="120" - echo -ne " ${INFO} Time until retry: ${secs}" - until timeout 1 getent hosts "${lookupDomain}" &>/dev/null; do - [[ "${secs:-}" -eq 0 ]] && break - echo -ne "${OVER} ${INFO} Time until retry: ${secs}" - : $((secs--)) + echo -e " ${INFO} Waiting until DNS resolution is available..." + until getent hosts github.com &> /dev/null; do + # Append one dot for each second waiting + str="${str}." + echo -ne " ${OVER} ${INFO} ${str}" sleep 1 done - # Try again - gravity_CheckDNSResolutionAvailable + # If we reach this point, DNS resolution is available + echo -e "${OVER} ${TICK} DNS resolution is available" } # Retrieve blocklist URLs and parse domains from adlist.list From fc73cee156b16c7b8b62594ad10103d3496b5cfe Mon Sep 17 00:00:00 2001 From: SkyLined Date: Mon, 8 Jul 2024 10:01:34 +0200 Subject: [PATCH 439/462] Fix risk of popd without a pushd `pushd` was inside if, `popd` was outside; there was a risk of not doing a `pushd` and only doing a `popd`. Signed-off-by: SkyLined --- automated install/basic-install.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 6b65313e..752e9816 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -378,14 +378,13 @@ is_repo() { pushd "${directory}" &>/dev/null || return 1 # Use git to check if the directory is a repo # git -C is not used here to support git versions older than 1.8.4 - git status --short &>/dev/null || rc=$? - # If the command was not successful, + git status --short &> /dev/null || rc=$? + # Move back into the directory the user started in + popd &> /dev/null || return 1 else # Set a non-zero return code if directory does not exist rc=1 fi - # Move back into the directory the user started in - popd &>/dev/null || return 1 # Return the code; if one is not set, return 0 return "${rc:-0}" } From 333764a7c1e4d8f6eceb7b6ba98525f6e994bd03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 4 Sep 2024 21:56:09 +0200 Subject: [PATCH 440/462] Account for renaming of `devel` branch on `web` repo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeCheckout.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 0514b8eb..84c966df 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -77,13 +77,13 @@ checkout() { if [[ "${1}" == "dev" ]] ; then # Shortcut to check out development branches - echo -e " ${INFO} Shortcut \"${COL_YELLOW}dev${COL_NC}\" detected - checking out development / devel branches..." + echo -e " ${INFO} Shortcut \"${COL_YELLOW}dev${COL_NC}\" detected - checking out development branches..." echo "" echo -e " ${INFO} Pi-hole Core" fetch_checkout_pull_branch "${PI_HOLE_FILES_DIR}" "development" || { echo " ${CROSS} Unable to pull Core development branch"; exit 1; } echo "" echo -e " ${INFO} Web interface" - fetch_checkout_pull_branch "${webInterfaceDir}" "devel" || { echo " ${CROSS} Unable to pull Web development branch"; exit 1; } + fetch_checkout_pull_branch "${webInterfaceDir}" "development" || { echo " ${CROSS} Unable to pull Web development branch"; exit 1; } #echo -e " ${TICK} Pi-hole Core" local path From cffb3117d4680e7b1bd600435c806f5bc2814f1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 4 Sep 2024 22:32:03 +0200 Subject: [PATCH 441/462] Remove lines containing ABP JavaScript rules from adlists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 1ac0e282..e7cd47cf 100755 --- a/gravity.sh +++ b/gravity.sh @@ -699,15 +699,17 @@ gravity_ParseFileIntoDomains() { # 3) Remove lines starting with ! (ABP Comments) # 4) Remove lines starting with [ (ABP Header) # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") preceded by a letter - # 6) Remove comments (text starting with "#", include possible spaces before the hash sign) - # 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) - # 8) Remove empty lines + # 6) Remove lines containing ABP JavaScript rules ("#%#") preceded by a letter + # 7) Remove comments (text starting with "#", include possible spaces before the hash sign) + # 8) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) + # 9) Remove empty lines sed -i -r \ -e 's/\r$//' \ -e 's/\s*!.*//g' \ -e 's/\s*\[.*//g' \ -e '/[a-z]\#[$?@]{0,1}\#/d' \ + -e '/[a-z]\#%\#/d' \ -e 's/\s*#.*//g' \ -e 's/^.*\s+//g' \ -e '/^$/d' "${destination}" From e291a9f7e044c0bb317c7f886c72648e8d43a163 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Thu, 5 Sep 2024 13:42:10 +0200 Subject: [PATCH 442/462] It's Adguard not ABP Co-authored-by: Adam Warner Signed-off-by: yubiuser --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index e7cd47cf..7d1358f2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -699,7 +699,7 @@ gravity_ParseFileIntoDomains() { # 3) Remove lines starting with ! (ABP Comments) # 4) Remove lines starting with [ (ABP Header) # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") preceded by a letter - # 6) Remove lines containing ABP JavaScript rules ("#%#") preceded by a letter + # 6) Remove lines containing Adguard JavaScript rules ("#%#") preceded by a letter # 7) Remove comments (text starting with "#", include possible spaces before the hash sign) # 8) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) # 9) Remove empty lines From 0ad2a79624a3f2365acabf9fe234e4325aa8e2fe Mon Sep 17 00:00:00 2001 From: yubiuser Date: Fri, 6 Sep 2024 21:20:36 +0200 Subject: [PATCH 443/462] Apply suggestions from code review Co-authored-by: Adam Warner Signed-off-by: yubiuser --- gravity.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 7d1358f2..dce387f5 100755 --- a/gravity.sh +++ b/gravity.sh @@ -699,7 +699,6 @@ gravity_ParseFileIntoDomains() { # 3) Remove lines starting with ! (ABP Comments) # 4) Remove lines starting with [ (ABP Header) # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") preceded by a letter - # 6) Remove lines containing Adguard JavaScript rules ("#%#") preceded by a letter # 7) Remove comments (text starting with "#", include possible spaces before the hash sign) # 8) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) # 9) Remove empty lines @@ -708,8 +707,7 @@ gravity_ParseFileIntoDomains() { -e 's/\r$//' \ -e 's/\s*!.*//g' \ -e 's/\s*\[.*//g' \ - -e '/[a-z]\#[$?@]{0,1}\#/d' \ - -e '/[a-z]\#%\#/d' \ + -e '/[a-z]\#[$?@%]{0,3}\#/d' \ -e 's/\s*#.*//g' \ -e 's/^.*\s+//g' \ -e '/^$/d' "${destination}" From f02162b0212ed1bb9e6cdaf0ae0dd9ae11e0a456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 6 Sep 2024 21:22:22 +0200 Subject: [PATCH 444/462] Update comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- gravity.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index dce387f5..4f345960 100755 --- a/gravity.sh +++ b/gravity.sh @@ -698,10 +698,10 @@ gravity_ParseFileIntoDomains() { # 2) Remove carriage returns # 3) Remove lines starting with ! (ABP Comments) # 4) Remove lines starting with [ (ABP Header) - # 5) Remove lines containing ABP extended CSS selectors ("##", "#!#", "#@#", "#?#") preceded by a letter - # 7) Remove comments (text starting with "#", include possible spaces before the hash sign) - # 8) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) - # 9) Remove empty lines + # 5) Remove lines containing ABP extended CSS selectors ("##", "#$#", "#@#", "#?#") and Adguard JavaScript (#%#) preceded by a letter + # 6) Remove comments (text starting with "#", include possible spaces before the hash sign) + # 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) + # 8) Remove empty lines sed -i -r \ -e 's/\r$//' \ From 129272a69553df0465d58fd354882c1a55bc3694 Mon Sep 17 00:00:00 2001 From: Jack'lul Date: Mon, 9 Sep 2024 18:37:17 +0200 Subject: [PATCH 445/462] Fix wrong message being displayed while waiting for DNS Signed-off-by: Jack'lul --- gravity.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 1ac0e282..7ab3243c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -304,7 +304,8 @@ gravity_CheckDNSResolutionAvailable() { echo -e " ${CROSS} DNS resolution is currently unavailable" fi - echo -e " ${INFO} Waiting until DNS resolution is available..." + str="Waiting until DNS resolution is available..." + echo -ne " ${INFO} ${str}" until getent hosts github.com &> /dev/null; do # Append one dot for each second waiting str="${str}." From d60ad57cac6a37bfe8f2ddb047601730833f52a5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 29 Aug 2024 20:01:06 +0200 Subject: [PATCH 446/462] Add /etc/pihole/dnsmasq.conf to debug log (stripped-down version) Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 7c558127..0a15972c 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -76,6 +76,7 @@ PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole.toml" +PIHOLE_DNSMASQ_CONF_FILE="${PIHOLE_DIRECTORY}/dnsmasq.conf" PIHOLE_VERSIONS_FILE="${PIHOLE_DIRECTORY}/versions" # Read the value of an FTL config key. The value is printed to stdout. @@ -114,6 +115,7 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}" "${PIHOLE_LOCAL_HOSTS_FILE}" "${PIHOLE_LOGROTATE_FILE}" "${PIHOLE_FTL_CONF_FILE}" +"${PIHOLE_DNSMASQ_CONF_FILE}" "${PIHOLE_COMMAND}" "${PIHOLE_COLTABLE_FILE}" "${FTL_PID}" From ce5ef79a3a7717c9f85e449658fc8d39c386fac3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Wed, 11 Sep 2024 13:31:47 +0200 Subject: [PATCH 447/462] Reduce code duplications and add check via IPv6 without hard-coded nameserver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 92 ++++++++++++++++++------------ 1 file changed, 56 insertions(+), 36 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 873d6d7c..f9cfc94c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -167,88 +167,108 @@ is_command() { command -v "${check_command}" >/dev/null 2>&1 } +os_check_dig(){ + local protocol="$1" + local domain="$2" + local nameserver="$3" + local response + + response="$(dig "${protocol}" +short -t txt "${domain}" "${nameserver}" 2>&1 + echo $? + )" + echo "${response}" +} + +os_check_dig_response(){ + # Checks the reply from the dig command to determine if it's a valid response + local digReply="$1" + local response + + # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid + response="${digReply%%$'\n'*}" + # If the value of ${response} is a single 0, then this is the return code, not an actual response. + if [ "${response}" == 0 ]; then + echo false + else + echo true + fi +} + os_check() { if [ "$PIHOLE_SKIP_OS_CHECK" != true ]; then # This function gets a list of supported OS versions from a TXT record at versions.pi-hole.net # and determines whether or not the script is running on one of those systems local remote_os_domain valid_os valid_version valid_response detected_os detected_version display_warning cmdResult digReturnCode response + local piholeNameserver="@ns1.pi-hole.net" remote_os_domain=${OS_CHECK_DOMAIN_NAME:-"versions.pi-hole.net"} detected_os=$(grep '^ID=' /etc/os-release | cut -d '=' -f2 | tr -d '"') detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') # Test via IPv4 and hardcoded nameserver ns1.pi-hole.net - cmdResult="$( - dig -4 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1 - echo $? - )" + cmdResult=$(os_check_dig 4 "${remote_os_domain}" "${piholeNameserver}") + # Gets the return code of the previous command (last line) digReturnCode="${cmdResult##*$'\n'}" if [ ! "${digReturnCode}" == "0" ]; then valid_response=false else - # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid - response="${cmdResult%%$'\n'*}" - # If the value of ${response} is a single 0, then this is the return code, not an actual response. - if [ "${response}" == 0 ]; then - valid_response=false - else - valid_response=true - fi + valid_response=$(os_check_dig_response cmdResult) fi # Try again via IPv6 and hardcoded nameserver ns1.pi-hole.net if [ "$valid_response" = false ]; then unset valid_response + unset cmdResult + unset digReturnCode - cmdResult="$( - dig -6 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1 - echo $? - )" + cmdResult=$(os_check_dig 6 "${remote_os_domain}" "${piholeNameserver}") # Gets the return code of the previous command (last line) digReturnCode="${cmdResult##*$'\n'}" if [ ! "${digReturnCode}" == "0" ]; then valid_response=false else - # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid - response="${cmdResult%%$'\n'*}" - # If the value of ${response} is a single 0, then this is the return code, not an actual response. - if [ "${response}" == 0 ]; then - valid_response=false - else - valid_response=true - fi + valid_response=$(os_check_dig_response cmdResult) fi fi # Try again without hardcoded nameserver if [ "$valid_response" = false ]; then unset valid_response + unset cmdResult + unset digReturnCode - cmdResult="$( - dig +short -t txt "${remote_os_domain}" 2>&1 - echo $? - )" + cmdResult=$(os_check_dig 4 "${remote_os_domain}") # Gets the return code of the previous command (last line) digReturnCode="${cmdResult##*$'\n'}" if [ ! "${digReturnCode}" == "0" ]; then valid_response=false else - # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid - response="${cmdResult%%$'\n'*}" - # If the value of ${response} is a single 0, then this is the return code, not an actual response. - if [ "${response}" == 0 ]; then - valid_response=false - else - valid_response=true - fi + valid_response=$(os_check_dig_response cmdResult) + fi + fi + + if [ "$valid_response" = false ]; then + unset valid_response + unset cmdResult + unset digReturnCode + + cmdResult=$(os_check_dig 6 "${remote_os_domain}") + # Gets the return code of the previous command (last line) + digReturnCode="${cmdResult##*$'\n'}" + + if [ ! "${digReturnCode}" == "0" ]; then + valid_response=false + else + valid_response=$(os_check_dig_response cmdResult) fi fi if [ "$valid_response" = true ]; then + response="${cmdResult%%$'\n'*}" IFS=" " read -r -a supportedOS < <(echo "${response}" | tr -d '"') for distro_and_versions in "${supportedOS[@]}"; do distro_part="${distro_and_versions%%=*}" From 62ef2d17772673713e483f12a7a291bec447a2bc Mon Sep 17 00:00:00 2001 From: yubiuser Date: Wed, 11 Sep 2024 18:44:17 +0200 Subject: [PATCH 448/462] Add forgetting dash Co-authored-by: RD WebDesign Signed-off-by: yubiuser --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f9cfc94c..18591773 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -173,7 +173,7 @@ os_check_dig(){ local nameserver="$3" local response - response="$(dig "${protocol}" +short -t txt "${domain}" "${nameserver}" 2>&1 + response="$(dig -"${protocol}" +short -t txt "${domain}" "${nameserver}" 2>&1 echo $? )" echo "${response}" From 3e8189e9ce1a34d3bdaa817e99a923d85693c49c Mon Sep 17 00:00:00 2001 From: Jim Bennett Date: Wed, 11 Sep 2024 15:00:57 -0700 Subject: [PATCH 449/462] Make install command copyable The instal command was in a header with inline markdown. This means there's no copy button on GitHub. Moving to a markdown block gives the copy button. Signed-off-by: Jim Bennett --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index eb50030b..7eb1fb3a 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,9 @@ The Pi-hole® is a [DNS sinkhole](https://en.wikipedia.org/wiki/DNS_Sinkhole) th Those who want to get started quickly and conveniently may install Pi-hole using the following command: -### `curl -sSL https://install.pi-hole.net | bash` +```bash +curl -sSL https://install.pi-hole.net | bash +``` ## Alternative Install Methods From 251f3295f3b64a81c86ef1e18ce92f91f8658f5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Thu, 12 Sep 2024 21:32:36 +0200 Subject: [PATCH 450/462] Return early during v6 migration if migration dir exists MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 752e9816..a123e8d9 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2146,8 +2146,8 @@ migrate_dnsmasq_configs() { # avoid conflicts with other services on this system # Exit early if this is already Pi-hole v6.0 - # We decide this on the presence of the file /etc/pihole/pihole.toml - if [[ -f /etc/pihole/pihole.toml ]]; then + # We decide this on the presence of the migration dir + if [[ -d "${V6_CONF_MIGRATION_DIR}" ]]; then return 0 fi From 68888979999657d9a0eb57bb319b1a415b6819b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 13 Sep 2024 21:15:25 +0200 Subject: [PATCH 451/462] Revert "Return early during v6 migration if migration dir exists" This reverts commit 251f3295f3b64a81c86ef1e18ce92f91f8658f5e. --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a123e8d9..752e9816 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2146,8 +2146,8 @@ migrate_dnsmasq_configs() { # avoid conflicts with other services on this system # Exit early if this is already Pi-hole v6.0 - # We decide this on the presence of the migration dir - if [[ -d "${V6_CONF_MIGRATION_DIR}" ]]; then + # We decide this on the presence of the file /etc/pihole/pihole.toml + if [[ -f /etc/pihole/pihole.toml ]]; then return 0 fi From 20d20e116c855b8d618caec1f94d8461f8157f36 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:25:32 +0000 Subject: [PATCH 452/462] Bump tox from 4.18.0 to 4.18.1 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.18.0 to 4.18.1. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.18.0...4.18.1) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 4bc7c372..2c417afb 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.2 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.18.0 +tox == 4.18.1 pytest-clarity == 1.0.1 From 05707c624d5e2b2e746ff9f32db8180b26ab3300 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 14 Sep 2024 10:36:07 +0000 Subject: [PATCH 453/462] Bump pytest from 8.3.2 to 8.3.3 in /test Bumps [pytest](https://github.com/pytest-dev/pytest) from 8.3.2 to 8.3.3. - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/8.3.2...8.3.3) --- updated-dependencies: - dependency-name: pytest dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 2c417afb..4b9882b7 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,5 +1,5 @@ pyyaml == 6.0.2 -pytest == 8.3.2 +pytest == 8.3.3 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 tox == 4.18.1 From 73301da68f0243717424cc3de7ffa6a0655168ce Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jan 2024 08:42:31 +0100 Subject: [PATCH 454/462] Optimize the database by running ANALYZE after gravity finished Signed-off-by: DL6ER --- gravity.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/gravity.sh b/gravity.sh index 7ab3243c..1a01946a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -837,6 +837,24 @@ database_recovery() { echo "" } +gravity_optimize() { + # The ANALYZE command gathers statistics about tables and indices and stores + # the collected information in internal tables of the database where the + # query optimizer can access the information and use it to help make better + # query planning choices + local str="Optimizing database" + echo -ne " ${INFO} ${str}..." + output=$( { pihole-FTL sqlite3 -ni "${gravityTEMPfile}" "PRAGMA analysis_limit=0; ANALYZE" 2>&1; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to optimize database ${gravityTEMPfile}\\n ${output}" + gravity_Cleanup "error" + else + echo -e "${OVER} ${TICK} ${str}" + fi +} + helpFunc() { echo "Usage: pihole -g Update domains from blocklists specified in adlists.list @@ -949,6 +967,9 @@ gravity_build_tree # numbers quickly from the tree instead of having to scan the whole database) gravity_ShowCount +# Optimize the database +gravity_optimize + # Migrate rest of the data from old to new database # IMPORTANT: Swapping the databases must be the last step before the cleanup if ! gravity_swap_databases; then From 111dfc63ff1d948de424ca31f1cb551dbe5310ac Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 17 Sep 2024 20:23:09 +0200 Subject: [PATCH 455/462] Add new option allowing timing the individual parts of gravity Signed-off-by: DL6ER --- gravity.sh | 65 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 9 deletions(-) diff --git a/gravity.sh b/gravity.sh index 1a01946a..05f3fb0c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -428,7 +428,7 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" "${domain}" + timeit gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" "${domain}" fi echo "" done @@ -855,12 +855,58 @@ gravity_optimize() { fi } +# Function: timeit +# Description: Measures the execution time of a given command. +# +# Usage: +# timeit +# +# Parameters: +# - The command to be executed and timed. +# +# Returns: +# The exit status of the executed command. +# +# Output: +# If the 'timed' variable is set to true, prints the elapsed time in seconds +# with millisecond precision. +# +# Example: +# timeit ls -l +# +timeit(){ + local start_time end_time elapsed_time ret + + # Capture the start time + start_time=$(date +%s%3N) + + # Execute the command passed as arguments + "$@" + ret=$? + + if [[ "${timed:-}" != true ]]; then + return $ret + fi + + # Capture the end time + end_time=$(date +%s%3N) + + # Calculate the elapsed time + elapsed_time=$((end_time - start_time)) + + # Display the elapsed time + printf " --> took %d.%03d seconds\n" $((elapsed_time / 1000)) $((elapsed_time % 1000)) + + return $ret +} + helpFunc() { echo "Usage: pihole -g Update domains from blocklists specified in adlists.list Options: -f, --force Force the download of all specified blocklists + -t, --timeit Time the gravity update process -h, --help Show this help dialog" exit 0 } @@ -897,6 +943,7 @@ Available options: for var in "$@"; do case "${var}" in "-f" | "--force") forceDelete=true ;; + "-t" | "--timeit") timed=true ;; "-r" | "--repair") repairSelector "$3" ;; "-u" | "--upgrade") upgrade_gravityDB "${gravityDBfile}" "${piholeDir}" @@ -925,11 +972,11 @@ if [[ "${recreate_database:-}" == true ]]; then fi if [[ "${recover_database:-}" == true ]]; then - database_recovery "$4" + timeit database_recovery "$4" fi # Move possibly existing legacy files to the gravity database -if ! migrate_to_database; then +if ! timeit migrate_to_database; then echo -e " ${CROSS} Unable to migrate to database. Please contact support." exit 1 fi @@ -943,7 +990,7 @@ if [[ "${forceDelete:-}" == true ]]; then fi # Gravity downloads blocklists next -if ! gravity_CheckDNSResolutionAvailable; then +if ! timeit gravity_CheckDNSResolutionAvailable; then echo -e " ${CROSS} Can not complete gravity update, no DNS is available. Please contact support." exit 1 fi @@ -961,23 +1008,23 @@ chown pihole:pihole "${gravityTEMPfile}" chmod g+w "${piholeDir}" "${gravityTEMPfile}" # Build the tree -gravity_build_tree +timeit gravity_build_tree # Compute numbers to be displayed (do this after building the tree to get the # numbers quickly from the tree instead of having to scan the whole database) -gravity_ShowCount +timeit gravity_ShowCount # Optimize the database -gravity_optimize +timeit gravity_optimize # Migrate rest of the data from old to new database # IMPORTANT: Swapping the databases must be the last step before the cleanup -if ! gravity_swap_databases; then +if ! timeit gravity_swap_databases; then echo -e " ${CROSS} Unable to create database. Please contact support." exit 1 fi -gravity_Cleanup +timeit gravity_Cleanup echo "" echo " ${TICK} Done." From 0f4ad7734c51ca3cde692d0cef759c95c5db36a8 Mon Sep 17 00:00:00 2001 From: Dominik Date: Wed, 18 Sep 2024 16:13:35 -0400 Subject: [PATCH 456/462] Update gravity.sh to use color with -t Co-authored-by: RD WebDesign Signed-off-by: Dominik --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 05f3fb0c..2605baf1 100755 --- a/gravity.sh +++ b/gravity.sh @@ -895,7 +895,7 @@ timeit(){ elapsed_time=$((end_time - start_time)) # Display the elapsed time - printf " --> took %d.%03d seconds\n" $((elapsed_time / 1000)) $((elapsed_time % 1000)) + printf " %b--> took %d.%03d seconds%b\n" ${COL_BLUE} $((elapsed_time / 1000)) $((elapsed_time % 1000)) ${COL_NC} return $ret } From 885895e9943de5da79116aaf2839bd82c7117723 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 21 Sep 2024 10:10:37 +0000 Subject: [PATCH 457/462] Bump tox from 4.18.1 to 4.20.0 in /test Bumps [tox](https://github.com/tox-dev/tox) from 4.18.1 to 4.20.0. - [Release notes](https://github.com/tox-dev/tox/releases) - [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst) - [Commits](https://github.com/tox-dev/tox/compare/4.18.1...4.20.0) --- updated-dependencies: - dependency-name: tox dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- test/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 4b9882b7..8f96a644 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -2,5 +2,5 @@ pyyaml == 6.0.2 pytest == 8.3.3 pytest-xdist == 3.6.1 pytest-testinfra == 10.1.1 -tox == 4.18.1 +tox == 4.20.0 pytest-clarity == 1.0.1 From 5f1e4680f7e07a2a721367d2c8daf21b25af5e4e Mon Sep 17 00:00:00 2001 From: Wouter Servaes Date: Tue, 10 Sep 2024 21:36:12 +0200 Subject: [PATCH 458/462] Grouped common dependencies of distros Signed-off-by: Wouter Servaes --- automated install/basic-install.sh | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 752e9816..b2803651 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -316,8 +316,10 @@ test_dpkg_lock() { # Compatibility package_manager_detect() { - # TODO - pull common packages for both distributions out into a common variable, then add - # the distro-specific ones below. + # pull common packages for both distributions out into a common variable + OS_CHECK_COMMON_DEPS=(grep) + PIHOLE_COMMON_DEPS=(curl psmisc sudo unzip jq); + INSTALLER_COMMON_DEPS=(git dialog ca-certificates) # First check to see if apt-get is installed. if is_command apt-get; then @@ -333,11 +335,11 @@ package_manager_detect() { # Update package cache update_package_cache || exit 1 # Packages required to perform the os_check and FTL binary detection - OS_CHECK_DEPS=(grep dnsutils binutils) + OS_CHECK_DEPS=(dnsutils binutils) # Packages required to run this install script - INSTALLER_DEPS=(git iproute2 dialog ca-certificates) + INSTALLER_DEPS=(iproute2) # Packages required to run Pi-hole - PIHOLE_DEPS=(cron curl iputils-ping psmisc sudo unzip libcap2-bin dns-root-data libcap2 netcat-openbsd procps jq lshw bash-completion) + PIHOLE_DEPS=(cron iputils-ping libcap2-bin dns-root-data libcap2 netcat-openbsd procps lshw bash-completion) # If apt-get is not found, check for rpm. elif is_command rpm; then @@ -352,9 +354,9 @@ package_manager_detect() { PKG_INSTALL=("${PKG_MANAGER}" install -y) # CentOS package manager returns 100 when there are packages to update so we need to || true to prevent the script from exiting. PKG_COUNT="${PKG_MANAGER} check-update | grep -E '(.i686|.x86|.noarch|.arm|.src|.riscv64)' | wc -l || true" - OS_CHECK_DEPS=(grep bind-utils) - INSTALLER_DEPS=(git dialog iproute newt procps-ng chkconfig ca-certificates binutils) - PIHOLE_DEPS=(cronie curl findutils sudo unzip psmisc libcap nmap-ncat jq lshw bash-completion) + OS_CHECK_DEPS=(bind-utils) + INSTALLER_DEPS=(iproute newt procps-ng chkconfig binutils) + PIHOLE_DEPS=(cronie findutils libcap nmap-ncat lshw bash-completion) # If neither apt-get or yum/dnf package managers were found else @@ -2223,7 +2225,7 @@ main() { # Install packages necessary to perform os_check printf " %b Checking for / installing Required dependencies for OS Check...\\n" "${INFO}" - install_dependent_packages "${OS_CHECK_DEPS[@]}" + install_dependent_packages "${OS_CHECK_COMMON_DEPS[@]}" "${OS_CHECK_DEPS[@]}" # Check that the installed OS is officially supported - display warning if not os_check @@ -2240,7 +2242,7 @@ main() { # Install packages used by this installation script printf " %b Checking for / installing Required dependencies for this install script...\\n" "${INFO}" - install_dependent_packages "${INSTALLER_DEPS[@]}" + install_dependent_packages "${INSTALLER_COMMON_DEPS[@]}" "${INSTALLER_DEPS[@]}" # if it's running unattended, if [[ "${runUnattended}" == true ]]; then @@ -2281,7 +2283,7 @@ main() { clone_or_update_repos # Install the Core dependencies - local dep_install_list=("${PIHOLE_DEPS[@]}") + local dep_install_list=("${PIHOLE_COMMON_DEPS[@]}" "${PIHOLE_DEPS[@]}") # Install packages used by the actual software printf " %b Checking for / installing Required dependencies for Pi-hole software...\\n" "${INFO}" From e9fdfac569e7a4476456cd66514752b4bff64977 Mon Sep 17 00:00:00 2001 From: Wouter Servaes Date: Wed, 11 Sep 2024 18:10:09 +0200 Subject: [PATCH 459/462] Added common dependencies to uninstall script for removal Signed-off-by: Wouter Servaes --- automated install/uninstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 8cffae20..ac06da73 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -45,7 +45,7 @@ source "${setupVars}" package_manager_detect # Uninstall packages used by the Pi-hole -DEPS=("${INSTALLER_DEPS[@]}" "${PIHOLE_DEPS[@]}" "${OS_CHECK_DEPS[@]}") +DEPS=("${INSTALLER_COMMON_DEPS[@]}" "${PIHOLE_COMMON_DEPS[@]}" "${OS_CHECK_COMMON_DEPS[@]}" "${INSTALLER_DEPS[@]}" "${PIHOLE_DEPS[@]}" "${OS_CHECK_DEPS[@]}") # Compatibility if [ -x "$(command -v apt-get)" ]; then From 8c56572d0b2eb82bd4854fa688287bcdd6b5df81 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 28 Sep 2024 10:12:44 +0000 Subject: [PATCH 460/462] Bump actions/checkout from 4.1.7 to 4.2.0 Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.7 to 4.2.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4.1.7...v4.2.0) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/sync-back-to-dev.yml | 2 +- .github/workflows/test.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 0a5b59b9..44c7ff5a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,7 +25,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v4.2.0 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 2a7831af..d2f552a2 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -40,7 +40,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v4.2.0 - name: Remove 'stale' label run: gh issue edit ${{ github.event.issue.number }} --remove-label ${{ env.stale_label }} env: diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml index fe24c9b5..067d4f2a 100644 --- a/.github/workflows/sync-back-to-dev.yml +++ b/.github/workflows/sync-back-to-dev.yml @@ -33,7 +33,7 @@ jobs: name: Syncing branches steps: - name: Checkout - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v4.2.0 - name: Opening pull request run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal' env: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0eef25f0..34a12b74 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v4.2.0 - name: Check scripts in repository are executable run: | @@ -70,7 +70,7 @@ jobs: DISTRO: ${{matrix.distro}} steps: - name: Checkout repository - uses: actions/checkout@v4.1.7 + uses: actions/checkout@v4.2.0 - name: Set up Python 3.10 uses: actions/setup-python@v5.2.0 From bcc3a7e35e9369f32b60a9c8abcf7187169a5880 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 30 Sep 2024 21:04:55 +0200 Subject: [PATCH 461/462] Show version information after an web only update MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/update.sh | 6 ++++++ automated install/basic-install.sh | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 8a35ef2e..2ccad27c 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -221,6 +221,12 @@ main() { echo -e " ${INFO} Local version file information updated." fi + # if there was only a web update, show the new versions + # (on core and FTL updates, this is done as part of the installer run) + if [[ "${web_update}" == true && "${FTL_update}" == false && "${core_update}" == false ]]; then + "${PI_HOLE_BIN_DIR}"/pihole version + fi + echo "" exit 0 } diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9428b05a..b948b8a5 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2434,7 +2434,7 @@ main() { if [[ "${INSTALL_TYPE}" == "Update" ]]; then printf "\\n" - "${PI_HOLE_BIN_DIR}"/pihole version --current + "${PI_HOLE_BIN_DIR}"/pihole version fi } From ada7d2bd31d36e5266d0a28e3773bccb13aac844 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 30 Sep 2024 21:10:33 +0200 Subject: [PATCH 462/462] Fix removing old man page MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9428b05a..ca81b6be 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1230,8 +1230,8 @@ install_manpage() { if [[ -f "/usr/local/share/man/man5/pihole-FTL.conf.5" ]]; then rm /usr/local/share/man/man5/pihole-FTL.conf.5 fi - if [[ -f "/usr/local/share/man/man5/pihole-FTL.8" ]]; then - rm /usr/local/share/man/man5/pihole-FTL.8 + if [[ -f "/usr/local/share/man/man8/pihole-FTL.8" ]]; then + rm /usr/local/share/man/man8/pihole-FTL.8 fi if mandb -q &>/dev/null; then