From 0de5132e2b2afef6e6fa1c573a9060c79d73d940 Mon Sep 17 00:00:00 2001 From: xch12i5 <40517505+xCh12i5@users.noreply.github.com> Date: Fri, 18 Jan 2019 22:14:34 +0100 Subject: [PATCH 001/207] Solves RegEx issue. Signed-off-by: xch12i5 <40517505+xCh12i5@users.noreply.github.com> --- gravity.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 75a51608..3247a8e2 100755 --- a/gravity.sh +++ b/gravity.sh @@ -338,11 +338,11 @@ gravity_ParseFileIntoDomains() { # Awk -F splits on given IFS, we grab the right hand side (chops trailing #coments and /'s to grab the domain only. # Last awk command takes non-commented lines and if they have 2 fields, take the right field (the domain) and leave # the left (IP address), otherwise grab the single field. - - < ${source} awk -F '#' '{print $1}' | \ - awk -F '/' '{print $1}' | \ - awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' | \ - sed -nr -e 's/\.{2,}/./g' -e '/\./p' > ${destination} + < ${source} tr -d '\r' | \ + tr '[:upper:]' '[:lower:]' | \ + sed -r '/(\/|#).*$/d' | \ + sed -r 's/^.*\s+//g' | \ + sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > ${destination} return 0 fi From e02da90faae124a9df3847b255f2116c126d26c7 Mon Sep 17 00:00:00 2001 From: xch12i5 <40517505+xCh12i5@users.noreply.github.com> Date: Sat, 19 Jan 2019 17:48:29 +0100 Subject: [PATCH 002/207] Replaces double quotes by single quotes and updates documentation. Signed-off-by: xch12i5 <40517505+xCh12i5@users.noreply.github.com> --- gravity.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 3247a8e2..7b9116a4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -335,14 +335,16 @@ gravity_ParseFileIntoDomains() { # Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious # This helps with that and makes it easier to read # It also helps with debugging so each stage of the script can be researched more in depth - # Awk -F splits on given IFS, we grab the right hand side (chops trailing #coments and /'s to grab the domain only. - # Last awk command takes non-commented lines and if they have 2 fields, take the right field (the domain) and leave - # the left (IP address), otherwise grab the single field. - < ${source} tr -d '\r' | \ - tr '[:upper:]' '[:lower:]' | \ - sed -r '/(\/|#).*$/d' | \ - sed -r 's/^.*\s+//g' | \ - sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > ${destination} + # 1) Remove carriage returns + # 2) Convert all characters to lowercase + # 3) Remove lines containing "#" or "/" + # 4) Remove leading tabs, spaces, etc. + # 5) Delete lines not matching domain names + < ${source} tr -d "\r" | \ + tr "[:upper:]" "[:lower:]" | \ + sed -r "/(\/|#).*$/d" | \ + sed -r "s/^.*\s+//g" | \ + sed -r "/([^\.]+\.)+[^\.]{2,}/!d" > ${destination} return 0 fi From cd8a423b32cd3900861085f9f64affc14cb544f2 Mon Sep 17 00:00:00 2001 From: xch12i5 <40517505+xCh12i5@users.noreply.github.com> Date: Sun, 20 Jan 2019 16:04:07 +0100 Subject: [PATCH 003/207] Solves RegEx issue and updates documentation. Signed-off-by: xch12i5 <40517505+xCh12i5@users.noreply.github.com> --- gravity.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 7b9116a4..6fe9be06 100755 --- a/gravity.sh +++ b/gravity.sh @@ -340,11 +340,11 @@ gravity_ParseFileIntoDomains() { # 3) Remove lines containing "#" or "/" # 4) Remove leading tabs, spaces, etc. # 5) Delete lines not matching domain names - < ${source} tr -d "\r" | \ - tr "[:upper:]" "[:lower:]" | \ - sed -r "/(\/|#).*$/d" | \ - sed -r "s/^.*\s+//g" | \ - sed -r "/([^\.]+\.)+[^\.]{2,}/!d" > ${destination} + < ${source} tr -d '\r' | \ + tr '[:upper:]' '[:lower:]' | \ + sed -r '/(\/|#).*$/d' | \ + sed -r 's/^.*\s+//g' | \ + sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > ${destination} return 0 fi From 09c4c88a6d1794a3ddb3ee8acb96e2cfb03ad7c2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 13:04:31 +0100 Subject: [PATCH 004/207] Create and fill gravity.db during pihole -g Signed-off-by: DL6ER --- advanced/Templates/gravity.db.schema | 13 +++++++ gravity.sh | 56 ++++++++++++++++++++++++++-- 2 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 advanced/Templates/gravity.db.schema diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema new file mode 100644 index 00000000..1eee8491 --- /dev/null +++ b/advanced/Templates/gravity.db.schema @@ -0,0 +1,13 @@ +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, comment TEXT, DateAdded DATETIME); + +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, comment TEXT, DateAdded DATETIME); + +CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); + +CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain +FROM gravity a +WHERE a.domain NOT IN (SELECT domain from whitelist); + +CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain +FROM blacklist a +WHERE a.domain NOT IN (SELECT domain from whitelist); diff --git a/gravity.sh b/gravity.sh index 75a51608..ac045a93 100755 --- a/gravity.sh +++ b/gravity.sh @@ -35,6 +35,10 @@ blackList="${piholeDir}/black.list" localList="${piholeDir}/local.list" VPNList="/etc/openvpn/ipp.txt" +piholeGitDir="/etc/.pihole" +gravityDBfile="${piholeDir}/gravity.db" +gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema" + domainsExtension="domains" matterAndLight="${basename}.0.matterandlight.txt" parsedMatter="${basename}.1.parsedmatter.txt" @@ -83,6 +87,11 @@ if [[ -r "${piholeDir}/pihole.conf" ]]; then echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}" fi +# Generate new sqlite3 file from schema template +generate_gravity_database() { + sqlite3 "${gravityDBfile}" < "${gravityDBschema}" +} + # Determine if Pi-hole blocking is disabled # If this is the case, we want to update # gravity.list.bck and black.list.bck instead of @@ -582,12 +591,38 @@ gravity_ParseBlacklistDomains() { cp "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}" fi - # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it - output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 ) + # Create database file if not present + if [ ! -e "${gravityDBfile}" ]; then + generate_gravity_database + fi + + # Backup gravity database + cp "${gravityDBfile}" "${gravityDBfile}.bck" + + # Empty domains + output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM gravity;"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n ${output}" + echo -e "\\n ${CROSS} Unable to truncate gravity database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi + + # Store domains in gravity database + output=$( { sqlite3 "${gravityDBfile}" <<< ".import ${piholeDir}/${accretionDisc} gravity"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to create gravity database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi + + # Empty $adList if it already exists, otherwise, create it + output=$( { : > "${adList}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to create empty ${adList}\\n ${output}" gravity_Cleanup "error" fi } @@ -633,6 +668,19 @@ gravity_Cleanup() { echo -e "${OVER} ${TICK} ${str}" + str="Optimizing domains database" + echo -ne " ${INFO} ${str}..." + # Store + output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + else + echo -e "${OVER} ${TICK} ${str}" + fi + # Only restart DNS service if offline if ! pidof ${resolver} &> /dev/null; then "${PIHOLE_COMMAND}" restartdns @@ -707,7 +755,7 @@ gravity_ShowBlockCount # Perform when downloading blocklists, or modifying the white/blacklist (not wildcards) if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then - str="Parsing domains into hosts format" + str="Parsing domains" echo -ne " ${INFO} ${str}..." gravity_ParseUserDomains From 8a91fe699276cdb1b1671d61f719653c197794b9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 13:21:26 +0100 Subject: [PATCH 005/207] Store domains in gravit tables: gravity, whitelist, blacklist Signed-off-by: DL6ER --- gravity.sh | 104 ++++++++++++++++++++++++++--------------------------- 1 file changed, 52 insertions(+), 52 deletions(-) diff --git a/gravity.sh b/gravity.sh index ac045a93..a8c92e8a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -33,6 +33,7 @@ regexFile="${piholeDir}/regex.list" adList="${piholeDir}/gravity.list" blackList="${piholeDir}/black.list" localList="${piholeDir}/local.list" +whiteList="${piholeDir}/white.list" VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" @@ -42,8 +43,6 @@ gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema" domainsExtension="domains" matterAndLight="${basename}.0.matterandlight.txt" parsedMatter="${basename}.1.parsedmatter.txt" -whitelistMatter="${basename}.2.whitelistmatter.txt" -accretionDisc="${basename}.3.accretionDisc.txt" preEventHorizon="list.preEventHorizon" skipDownload="false" @@ -92,6 +91,49 @@ generate_gravity_database() { sqlite3 "${gravityDBfile}" < "${gravityDBschema}" } +# Import domains from file and store them in the specified database table +gravity_store_in_database() { + # Define locals + local table="${1}" + local source="${2}" + local template="${3}" + + # Create database file if not present + if [ ! -e "${gravityDBfile}" ]; then + generate_gravity_database + fi + + # Backup gravity database + cp "${gravityDBfile}" "${gravityDBfile}.bck" + + # Empty domains + output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to truncate ${table} database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi + + # Store domains in gravity database + output=$( { sqlite3 "${gravityDBfile}" <<< ".import \"${source}\" ${table}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to create ${table} database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi + + # Empty $adList if it already exists, otherwise, create it + output=$( { : > "${template}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to create empty ${template}\\n ${output}" + gravity_Cleanup "error" + fi +} + # Determine if Pi-hole blocking is disabled # If this is the case, we want to update # gravity.list.bck and black.list.bck instead of @@ -476,7 +518,7 @@ gravity_SortAndFilterConsolidatedList() { echo -ne " ${INFO} ${str}..." fi - # Parse into hosts file + # Parse into file gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}" # Format $parsedMatter line total as currency @@ -514,8 +556,8 @@ gravity_Whitelist() { str="Number of whitelisted domains: ${num}" echo -ne " ${INFO} ${str}..." - # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile - comm -23 "${piholeDir}/${preEventHorizon}" <(sort "${whitelistFile}") > "${piholeDir}/${whitelistMatter}" + # Store whitelisted files in gravity database + gravity_store_in_database "whitelist" "${whitelistFile}" "${whiteList}" echo -e "${OVER} ${INFO} ${str}" } @@ -581,50 +623,8 @@ gravity_ParseLocalDomains() { gravity_ParseBlacklistDomains() { local output status - # Empty $accretionDisc if it already exists, otherwise, create it - : > "${piholeDir}/${accretionDisc}" - - if [[ -f "${piholeDir}/${whitelistMatter}" ]]; then - mv "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}" - else - # There was no whitelist file, so use preEventHorizon instead of whitelistMatter. - cp "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}" - fi - - # Create database file if not present - if [ ! -e "${gravityDBfile}" ]; then - generate_gravity_database - fi - - # Backup gravity database - cp "${gravityDBfile}" "${gravityDBfile}.bck" - - # Empty domains - output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM gravity;"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to truncate gravity database ${gravityDBfile}\\n ${output}" - gravity_Cleanup "error" - fi - - # Store domains in gravity database - output=$( { sqlite3 "${gravityDBfile}" <<< ".import ${piholeDir}/${accretionDisc} gravity"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to create gravity database ${gravityDBfile}\\n ${output}" - gravity_Cleanup "error" - fi - - # Empty $adList if it already exists, otherwise, create it - output=$( { : > "${adList}"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to create empty ${adList}\\n ${output}" - gravity_Cleanup "error" - fi + # Store gravity domains in gravity database + gravity_store_in_database "gravity" "${piholeDir}/${preEventHorizon}" "${adList}" } # Create user-added blacklist entries @@ -632,9 +632,9 @@ gravity_ParseUserDomains() { if [[ ! -f "${blacklistFile}" ]]; then return 0 fi - # Copy the file over as /etc/pihole/black.list so dnsmasq can use it - cp "${blacklistFile}" "${blackList}" 2> /dev/null || \ - echo -e "\\n ${CROSS} Unable to move ${blacklistFile##*/} to ${piholeDir}" + + # Fill database table + gravity_store_in_database "blacklist" "${blacklistFile}" "${blackList}" } # Trap Ctrl-C From 710036adae672486abf35be122a5d571d681d428 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 13:34:08 +0100 Subject: [PATCH 006/207] Add DISABLED field to white- and blacklist defaulting to false. The two views respect the DISABLED fields for both lists. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.schema | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema index 1eee8491..2fd9a9d1 100644 --- a/advanced/Templates/gravity.db.schema +++ b/advanced/Templates/gravity.db.schema @@ -1,13 +1,14 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, comment TEXT, DateAdded DATETIME); +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, comment TEXT, disabled BOOLEAN DEFAULT 0, DateAdded DATETIME); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, comment TEXT, DateAdded DATETIME); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, comment TEXT, disabled BOOLEAN DEFAULT 0, DateAdded DATETIME); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain FROM gravity a -WHERE a.domain NOT IN (SELECT domain from whitelist); +WHERE a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1); CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain FROM blacklist a -WHERE a.domain NOT IN (SELECT domain from whitelist); +WHERE a.disabled != 1 AND + a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1); From dcf0a605cf1b6e101b34206a3d5bd359f3c331a5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 14:14:14 +0100 Subject: [PATCH 007/207] Use views for all tables and set disabled column to false for those tables that support it Signed-off-by: DL6ER --- advanced/Templates/gravity.db.schema | 10 ++++++---- gravity.sh | 17 ++++++++++++++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema index 2fd9a9d1..83141f94 100644 --- a/advanced/Templates/gravity.db.schema +++ b/advanced/Templates/gravity.db.schema @@ -1,7 +1,5 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, comment TEXT, disabled BOOLEAN DEFAULT 0, DateAdded DATETIME); - -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, comment TEXT, disabled BOOLEAN DEFAULT 0, DateAdded DATETIME); - +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain @@ -12,3 +10,7 @@ CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain FROM blacklist a WHERE a.disabled != 1 AND a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1); + +CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain +FROM whitelist a +WHERE a.disabled != 1; diff --git a/gravity.sh b/gravity.sh index a8c92e8a..21338119 100755 --- a/gravity.sh +++ b/gravity.sh @@ -115,16 +115,27 @@ gravity_store_in_database() { gravity_Cleanup "error" fi - # Store domains in gravity database + # Store domains in gravity database table ${table} output=$( { sqlite3 "${gravityDBfile}" <<< ".import \"${source}\" ${table}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to create ${table} database ${gravityDBfile}\\n ${output}" + echo -e "\\n ${CROSS} Unable to create ${table} in database ${gravityDBfile}\\n ${output}" gravity_Cleanup "error" fi - # Empty $adList if it already exists, otherwise, create it + if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ]; then + # Set disabled to false + output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET disabled = 0 WHERE disabled IS NULL;"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to set disabled states (${table}) in database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi + fi + + # Empty $template if it already exists, otherwise, create it output=$( { : > "${template}"; } 2>&1 ) status="$?" From 93f1859babd7a28de799edb84fdeadf39300daba Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 15:13:18 +0100 Subject: [PATCH 008/207] Store regex in gravity.db as well Signed-off-by: DL6ER --- advanced/Templates/gravity.db.schema | 5 +++++ gravity.sh | 20 +++++++++++++------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema index 83141f94..b7b3808a 100644 --- a/advanced/Templates/gravity.db.schema +++ b/advanced/Templates/gravity.db.schema @@ -1,5 +1,6 @@ CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); +CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain @@ -14,3 +15,7 @@ WHERE a.disabled != 1 AND CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain FROM whitelist a WHERE a.disabled != 1; + +CREATE VIEW vw_regex AS SELECT DISTINCT a.domain +FROM regex a +WHERE a.disabled != 1; diff --git a/gravity.sh b/gravity.sh index 21338119..e3e78528 100755 --- a/gravity.sh +++ b/gravity.sh @@ -124,7 +124,7 @@ gravity_store_in_database() { gravity_Cleanup "error" fi - if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ]; then + if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then # Set disabled to false output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET disabled = 0 WHERE disabled IS NULL;"; } 2>&1 ) status="$?" @@ -135,13 +135,16 @@ gravity_store_in_database() { fi fi - # Empty $template if it already exists, otherwise, create it - output=$( { : > "${template}"; } 2>&1 ) - status="$?" + # Only create template file if asked to + if [ "${template}" != "-" ]; then + # Empty $template if it already exists, otherwise, create it + output=$( { : > "${template}"; } 2>&1 ) + status="$?" - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to create empty ${template}\\n ${output}" - gravity_Cleanup "error" + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to create empty ${template}\\n ${output}" + gravity_Cleanup "error" + fi fi } @@ -586,6 +589,9 @@ gravity_ShowBlockCount() { num=$(grep -cv "^#" "${regexFile}") echo -e " ${INFO} Number of regex filters: ${num}" fi + + # Store regex files in gravity database + gravity_store_in_database "regex" "${regexFile}" "-" } # Parse list of domains into hosts format From fc62cf7e2f043cc28d163386aaf7cd0134dce306 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 16:44:05 +0100 Subject: [PATCH 009/207] Only run VACUUM on "full" gravity runs Signed-off-by: DL6ER --- gravity.sh | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/gravity.sh b/gravity.sh index e3e78528..0a1d4e8d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -39,6 +39,7 @@ VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" gravityDBfile="${piholeDir}/gravity.db" gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema" +optimize_database=true domainsExtension="domains" matterAndLight="${basename}.0.matterandlight.txt" @@ -685,17 +686,19 @@ gravity_Cleanup() { echo -e "${OVER} ${TICK} ${str}" - str="Optimizing domains database" - echo -ne " ${INFO} ${str}..." - # Store - output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 ) - status="$?" + if ${optimize_database} ; then + str="Optimizing domains database" + echo -ne " ${INFO} ${str}..." + # Store + output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 ) + status="$?" - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n ${output}" - gravity_Cleanup "error" - else - echo -e "${OVER} ${TICK} ${str}" + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + else + echo -e "${OVER} ${TICK} ${str}" + fi fi # Only restart DNS service if offline @@ -726,9 +729,9 @@ for var in "$@"; do "-f" | "--force" ) forceDelete=true;; "-h" | "--help" ) helpFunc;; "-sd" | "--skip-download" ) skipDownload=true;; - "-b" | "--blacklist-only" ) listType="blacklist";; - "-w" | "--whitelist-only" ) listType="whitelist";; - "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";; + "-b" | "--blacklist-only" ) listType="blacklist"; optimize_database=false;; + "-w" | "--whitelist-only" ) listType="whitelist"; optimize_database=false;; + "-wild" | "--wildcard-only" ) listType="wildcard"; optimize_database=false; dnsRestartType="restart";; esac done From d5feffa117c2093e8dbd17f814bf95a7167ab2ba Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 16:45:19 +0100 Subject: [PATCH 010/207] Do not backup the database Signed-off-by: DL6ER --- gravity.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 0a1d4e8d..0d312b3f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -104,9 +104,6 @@ gravity_store_in_database() { generate_gravity_database fi - # Backup gravity database - cp "${gravityDBfile}" "${gravityDBfile}.bck" - # Empty domains output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 ) status="$?" From 1f9a7d0b0cdba219cc2e3ad7b5eb91e744ab915b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 17:01:38 +0100 Subject: [PATCH 011/207] Only run VACUUM on "pihole -g -o" Signed-off-by: DL6ER --- gravity.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/gravity.sh b/gravity.sh index 0d312b3f..d0b44c24 100755 --- a/gravity.sh +++ b/gravity.sh @@ -39,7 +39,7 @@ VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" gravityDBfile="${piholeDir}/gravity.db" gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema" -optimize_database=true +optimize_database=false domainsExtension="domains" matterAndLight="${basename}.0.matterandlight.txt" @@ -724,11 +724,12 @@ Options: for var in "$@"; do case "${var}" in "-f" | "--force" ) forceDelete=true;; + "-o" | "--optimize" ) optimize_database=true;; "-h" | "--help" ) helpFunc;; "-sd" | "--skip-download" ) skipDownload=true;; - "-b" | "--blacklist-only" ) listType="blacklist"; optimize_database=false;; - "-w" | "--whitelist-only" ) listType="whitelist"; optimize_database=false;; - "-wild" | "--wildcard-only" ) listType="wildcard"; optimize_database=false; dnsRestartType="restart";; + "-b" | "--blacklist-only" ) listType="blacklist";; + "-w" | "--whitelist-only" ) listType="whitelist";; + "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";; esac done From b2f4385232f7293d16e2a24eee6b60e3e7011308 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 3 Feb 2019 17:05:00 +0100 Subject: [PATCH 012/207] Avoid infinity loop if gravity optimization fails Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index d0b44c24..06e75a90 100755 --- a/gravity.sh +++ b/gravity.sh @@ -692,7 +692,7 @@ gravity_Cleanup() { if [[ "${status}" -ne 0 ]]; then echo -e "\\n ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n ${output}" - gravity_Cleanup "error" + error="error" else echo -e "${OVER} ${TICK} ${str}" fi From dbbf21071b4d3a13522d63bdb873ead883f2e13f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 5 Feb 2019 19:05:11 +0100 Subject: [PATCH 013/207] Invert logic to use ENABLED instead of DISABLED. We make sure to set "enabled = 1 WHERE enabled IS NULL" in gravity_store_in_database() Signed-off-by: DL6ER --- advanced/Templates/gravity.db.schema | 21 --------------------- advanced/Templates/gravity.db.sql | 21 +++++++++++++++++++++ gravity.sh | 8 ++++---- 3 files changed, 25 insertions(+), 25 deletions(-) delete mode 100644 advanced/Templates/gravity.db.schema create mode 100644 advanced/Templates/gravity.db.sql diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema deleted file mode 100644 index b7b3808a..00000000 --- a/advanced/Templates/gravity.db.schema +++ /dev/null @@ -1,21 +0,0 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); -CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME); -CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); - -CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain -FROM gravity a -WHERE a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1); - -CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain -FROM blacklist a -WHERE a.disabled != 1 AND - a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1); - -CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain -FROM whitelist a -WHERE a.disabled != 1; - -CREATE VIEW vw_regex AS SELECT DISTINCT a.domain -FROM regex a -WHERE a.disabled != 1; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql new file mode 100644 index 00000000..72d64bc1 --- /dev/null +++ b/advanced/Templates/gravity.db.sql @@ -0,0 +1,21 @@ +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME); +CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME); +CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); + +CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain +FROM gravity a +WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); + +CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain +FROM blacklist a +WHERE a.enabled == 1 AND + a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); + +CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain +FROM whitelist a +WHERE a.enabled == 1; + +CREATE VIEW vw_regex AS SELECT DISTINCT a.domain +FROM regex a +WHERE a.enabled == 1; diff --git a/gravity.sh b/gravity.sh index 06e75a90..c47f7487 100755 --- a/gravity.sh +++ b/gravity.sh @@ -38,7 +38,7 @@ VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" gravityDBfile="${piholeDir}/gravity.db" -gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema" +gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" optimize_database=false domainsExtension="domains" @@ -123,12 +123,12 @@ gravity_store_in_database() { fi if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then - # Set disabled to false - output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET disabled = 0 WHERE disabled IS NULL;"; } 2>&1 ) + # Set enabled to true where it is unspecified + output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET enabled = 1 WHERE enabled IS NULL;"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to set disabled states (${table}) in database ${gravityDBfile}\\n ${output}" + echo -e "\\n ${CROSS} Unable to set enabled states (${table}) in database ${gravityDBfile}\\n ${output}" gravity_Cleanup "error" fi fi From 8a2363621d084bff58ab160cc01633ae32aa4f19 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 5 Feb 2019 19:06:46 +0100 Subject: [PATCH 014/207] Rename fields from DateAdded to dateadded Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 72d64bc1..4470c21c 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,6 +1,6 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME); -CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME); +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); +CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain From c3c60e10f18db58e6a3cb09beb573a4eef26f8c8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 5 Feb 2019 19:07:12 +0100 Subject: [PATCH 015/207] Rename regex field from domain to filter because this described better what it is Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 4470c21c..5290e83b 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,6 +1,6 @@ CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); -CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); +CREATE TABLE regex (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain From ec5b16ef851e25167997fe02fd23944be65c7b4d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 5 Feb 2019 19:08:08 +0100 Subject: [PATCH 016/207] Fix incomplete comment Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index c47f7487..c45e124e 100755 --- a/gravity.sh +++ b/gravity.sh @@ -686,7 +686,7 @@ gravity_Cleanup() { if ${optimize_database} ; then str="Optimizing domains database" echo -ne " ${INFO} ${str}..." - # Store + # Run VACUUM command on database to optimize it output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 ) status="$?" From 9984647ebbc053f21881c8e4473a4ac5220050cf Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 5 Feb 2019 19:41:11 +0100 Subject: [PATCH 017/207] After renaming column in regex, it also needs to be renamed in vw_regex Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 5290e83b..23efa64a 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -16,6 +16,6 @@ CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain FROM whitelist a WHERE a.enabled == 1; -CREATE VIEW vw_regex AS SELECT DISTINCT a.domain +CREATE VIEW vw_regex AS SELECT DISTINCT a.filter FROM regex a WHERE a.enabled == 1; From 644ec36e6462a9b46f2e63fc5185040e3a803ebd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 6 Feb 2019 18:57:48 +0100 Subject: [PATCH 018/207] Review comments Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 9 +++++--- gravity.sh | 34 ++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 23efa64a..3a2230c8 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,7 +1,10 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); -CREATE TABLE regex (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME); +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); +CREATE TABLE regex (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); +CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); + +INSERT INTO info VALUES("version","1"); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain FROM gravity a diff --git a/gravity.sh b/gravity.sh index c45e124e..0c3c8dee 100755 --- a/gravity.sh +++ b/gravity.sh @@ -20,6 +20,8 @@ source "${regexconverter}" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" +PIHOLE_USER="pihole" +PIHOLE_GROUP="pihole" piholeDir="/etc/${basename}" @@ -90,6 +92,7 @@ fi # Generate new sqlite3 file from schema template generate_gravity_database() { sqlite3 "${gravityDBfile}" < "${gravityDBschema}" + chown $PIHOLE_USER:$PIHOLE_GROUP "${gravityDBfile}" } # Import domains from file and store them in the specified database table @@ -113,25 +116,32 @@ gravity_store_in_database() { gravity_Cleanup "error" fi + local tmpFile=$(mktemp -p "/tmp" --suffix=".gravity") + if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then + # Apply format for white-, blacklist, and regex tables, prevent globbing + set -f + for domain in $(cat < "${source}"); do + echo "\"${domain}\",1,$(date --utc +'%s')," >> "${tmpFile}" + done + set +f + inputfile="${tmpFile}" + else + # No need to modify the input data for the gravity table + inputfile="${source}" + fi # Store domains in gravity database table ${table} - output=$( { sqlite3 "${gravityDBfile}" <<< ".import \"${source}\" ${table}"; } 2>&1 ) + # Use printf as .mode and .import need to be on separate lines + # see https://unix.stackexchange.com/a/445615/83260 + output=$( { printf ".mode csv\n.import \"${inputfile}\" ${table}\n" | sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to create ${table} in database ${gravityDBfile}\\n ${output}" + echo -e "\\n ${CROSS} Unable to fill table ${table} in database ${gravityDBfile}\\n ${output}" gravity_Cleanup "error" fi - if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then - # Set enabled to true where it is unspecified - output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET enabled = 1 WHERE enabled IS NULL;"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to set enabled states (${table}) in database ${gravityDBfile}\\n ${output}" - gravity_Cleanup "error" - fi - fi + # Delete tmpfile + #rm "$tmpFile" > /dev/null 2>&1 || true # Only create template file if asked to if [ "${template}" != "-" ]; then From 491c828661ba203e679acc74df48a22970f2aee5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 6 Feb 2019 19:09:09 +0100 Subject: [PATCH 019/207] Improved code, ensure that gravity.sh passes shellcheck as a whole Signed-off-by: DL6ER --- gravity.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/gravity.sh b/gravity.sh index 0c3c8dee..129d0ed4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -116,14 +116,15 @@ gravity_store_in_database() { gravity_Cleanup "error" fi - local tmpFile=$(mktemp -p "/tmp" --suffix=".gravity") + local tmpFile + tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")" if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then - # Apply format for white-, blacklist, and regex tables, prevent globbing - set -f - for domain in $(cat < "${source}"); do + # Apply format for white-, blacklist, and regex tables + # Read file line by line + grep -v '^ *#' < "${source}" | while IFS= read -r domain + do echo "\"${domain}\",1,$(date --utc +'%s')," >> "${tmpFile}" done - set +f inputfile="${tmpFile}" else # No need to modify the input data for the gravity table @@ -132,7 +133,7 @@ gravity_store_in_database() { # Store domains in gravity database table ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".mode csv\n.import \"${inputfile}\" ${table}\n" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".mode csv\n.import \"%s\" ${table}\n" "${inputfile}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -412,10 +413,10 @@ gravity_ParseFileIntoDomains() { # Last awk command takes non-commented lines and if they have 2 fields, take the right field (the domain) and leave # the left (IP address), otherwise grab the single field. - < ${source} awk -F '#' '{print $1}' | \ + < "${source}" awk -F '#' '{print $1}' | \ awk -F '/' '{print $1}' | \ awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' | \ - sed -nr -e 's/\.{2,}/./g' -e '/\./p' > ${destination} + sed -nr -e 's/\.{2,}/./g' -e '/\./p' > "${destination}" return 0 fi From f9250d91a52a0bf2c62223ed51b96acf4a80b692 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 6 Feb 2019 19:13:31 +0100 Subject: [PATCH 020/207] Explictly escape the escape. Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 129d0ed4..c742a699 100755 --- a/gravity.sh +++ b/gravity.sh @@ -133,7 +133,7 @@ gravity_store_in_database() { # Store domains in gravity database table ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".mode csv\n.import \"%s\" ${table}\n" "${inputfile}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".mode csv\\n.import \"%s\" ${table}\\n" "${inputfile}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then From 61840edb301005178101ed821da01b05a22e1c0d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 22 Feb 2019 22:46:19 +0100 Subject: [PATCH 021/207] Fill comment field with NULL (default) instead of empty string and use a variable for the timestamp instead of periodically calling date Signed-off-by: DL6ER --- gravity.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index c742a699..6ace8108 100755 --- a/gravity.sh +++ b/gravity.sh @@ -118,12 +118,14 @@ gravity_store_in_database() { local tmpFile tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")" + local timestamp + timestamp="$(date --utc +'%s')" if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then # Apply format for white-, blacklist, and regex tables # Read file line by line grep -v '^ *#' < "${source}" | while IFS= read -r domain do - echo "\"${domain}\",1,$(date --utc +'%s')," >> "${tmpFile}" + echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}" done inputfile="${tmpFile}" else From bbc392caeb631e46f840d1e87d6bebe07442545a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 22 Feb 2019 22:49:02 +0100 Subject: [PATCH 022/207] More review comments Signed-off-by: DL6ER --- gravity.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 6ace8108..b09ddc0c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -120,6 +120,7 @@ gravity_store_in_database() { tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")" local timestamp timestamp="$(date --utc +'%s')" + local inputfile if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then # Apply format for white-, blacklist, and regex tables # Read file line by line @@ -135,7 +136,7 @@ gravity_store_in_database() { # Store domains in gravity database table ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".mode csv\\n.import \"%s\" ${table}\\n" "${inputfile}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".mode csv\\n.import \"%s\" %s\\n" "${inputfile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -144,7 +145,7 @@ gravity_store_in_database() { fi # Delete tmpfile - #rm "$tmpFile" > /dev/null 2>&1 || true + rm "$tmpFile" > /dev/null 2>&1 || true # Only create template file if asked to if [ "${template}" != "-" ]; then From 54ae9d83b0a4ff09c2a40db079df9952cfd122af Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Sat, 23 Feb 2019 13:42:15 -0800 Subject: [PATCH 023/207] Stickler-lints Signed-off-by: Dan Schaper --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index 6fe9be06..015764ac 100755 --- a/gravity.sh +++ b/gravity.sh @@ -340,11 +340,11 @@ gravity_ParseFileIntoDomains() { # 3) Remove lines containing "#" or "/" # 4) Remove leading tabs, spaces, etc. # 5) Delete lines not matching domain names - < ${source} tr -d '\r' | \ + < "${source}" tr -d '\r' | \ tr '[:upper:]' '[:lower:]' | \ sed -r '/(\/|#).*$/d' | \ sed -r 's/^.*\s+//g' | \ - sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > ${destination} + sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > "${destination}" return 0 fi From de4456330169a3328c6e22ab4860a4f42a916c22 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 18:51:38 +0200 Subject: [PATCH 024/207] Remove addn-hosts from 01-pihole.conf Signed-off-by: DL6ER --- advanced/01-pihole.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf index 502293bf..40a117fe 100644 --- a/advanced/01-pihole.conf +++ b/advanced/01-pihole.conf @@ -18,8 +18,6 @@ # WITHIN /etc/dnsmasq.d/yourname.conf # ############################################################################### -addn-hosts=/etc/pihole/gravity.list -addn-hosts=/etc/pihole/black.list addn-hosts=/etc/pihole/local.list domain-needed From 6594a0a6e75ed60c5b15f422a9b319f14e312330 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 18:53:20 +0200 Subject: [PATCH 025/207] Fix detection of blocked query in pihole -t Signed-off-by: DL6ER --- pihole | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pihole b/pihole index 84a5623f..6b572fa9 100755 --- a/pihole +++ b/pihole @@ -313,7 +313,7 @@ tailFunc() { # Colour everything else as gray tail -f /var/log/pihole.log | sed -E \ -e "s,($(date +'%b %d ')| dnsmasq[.*[0-9]]),,g" \ - -e "s,(.*(gravity.list|black.list|regex.list| config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \ + -e "s,(.*(gravity |black |regex | config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \ -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \ -e "s,.*,${COL_GRAY}&${COL_NC}," exit 0 From c13ebd3a9d9cab33e0d6e8d389b5e69e8115d14d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 18:54:55 +0200 Subject: [PATCH 026/207] Replace Pi-hole ID string with another line we intend to keep Signed-off-by: DL6ER --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 110399f8..ffc1bdf6 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1224,7 +1224,7 @@ version_check_dnsmasq() { # Local, named variables local dnsmasq_conf="/etc/dnsmasq.conf" local dnsmasq_conf_orig="/etc/dnsmasq.conf.orig" - local dnsmasq_pihole_id_string="addn-hosts=/etc/pihole/gravity.list" + local dnsmasq_pihole_id_string="# Dnsmasq config for Pi-hole's FTLDNS" local dnsmasq_original_config="${PI_HOLE_LOCAL_REPO}/advanced/dnsmasq.conf.original" local dnsmasq_pihole_01_snippet="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf" local dnsmasq_pihole_01_location="/etc/dnsmasq.d/01-pihole.conf" @@ -1232,7 +1232,7 @@ version_check_dnsmasq() { # If the dnsmasq config file exists if [[ -f "${dnsmasq_conf}" ]]; then printf " %b Existing dnsmasq.conf found..." "${INFO}" - # If gravity.list is found within this file, we presume it's from older versions on Pi-hole, + # If a specific string is found within this file, we presume it's from older versions on Pi-hole, if grep -q ${dnsmasq_pihole_id_string} ${dnsmasq_conf}; then printf " it is from a previous Pi-hole install.\\n" printf " %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}" From d7ee44960e25b3c3bb89c2e81578c1a8cff14087 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 18:56:44 +0200 Subject: [PATCH 027/207] Remove moving around of the gravity and blacklist files as we do not use them any longer Signed-off-by: DL6ER --- pihole | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/pihole b/pihole index 6b572fa9..71c286c6 100755 --- a/pihole +++ b/pihole @@ -10,10 +10,8 @@ # Please see LICENSE file for your rights under this license. readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" -readonly gravitylist="/etc/pihole/gravity.list" -readonly blacklist="/etc/pihole/black.list" -# setupVars is not readonly here because in some funcitons (checkout), +# setupVars is not readonly here because in some functions (checkout), # it might get set again when the installer is sourced. This causes an # error due to modifying a readonly variable. setupVars="/etc/pihole/setupVars.conf" @@ -148,14 +146,6 @@ Time: echo -e " ${INFO} Blocking already disabled, nothing to do" exit 0 fi - if [[ -e "${gravitylist}" ]]; then - mv "${gravitylist}" "${gravitylist}.bck" - echo "" > "${gravitylist}" - fi - if [[ -e "${blacklist}" ]]; then - mv "${blacklist}" "${blacklist}.bck" - echo "" > "${blacklist}" - fi if [[ $# > 1 ]]; then local error=false if [[ "${2}" == *"s" ]]; then @@ -204,12 +194,6 @@ Time: echo -e " ${INFO} Enabling blocking" local str="Pi-hole Enabled" - if [[ -e "${gravitylist}.bck" ]]; then - mv "${gravitylist}.bck" "${gravitylist}" - fi - if [[ -e "${blacklist}.bck" ]]; then - mv "${blacklist}.bck" "${blacklist}" - fi sed -i "/BLOCKING_ENABLED=/d" "${setupVars}" echo "BLOCKING_ENABLED=true" >> "${setupVars}" fi From ee20164207508ca1e2dbb08e05b0858e074dd370 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 19:01:31 +0200 Subject: [PATCH 028/207] Remove detect_pihole_blocking_status() as we can modify the database at any time Signed-off-by: DL6ER --- gravity.sh | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/gravity.sh b/gravity.sh index b09ddc0c..b287653d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -160,20 +160,6 @@ gravity_store_in_database() { fi } -# Determine if Pi-hole blocking is disabled -# If this is the case, we want to update -# gravity.list.bck and black.list.bck instead of -# gravity.list and black.list -detect_pihole_blocking_status() { - if [[ "${BLOCKING_ENABLED}" == false ]]; then - echo -e " ${INFO} Pi-hole blocking is disabled" - adList="${adList}.bck" - blackList="${blackList}.bck" - else - echo -e " ${INFO} Pi-hole blocking is enabled" - fi -} - # Determine if DNS resolution is available before proceeding gravity_CheckDNSResolutionAvailable() { local lookupDomain="pi.hole" @@ -235,7 +221,7 @@ gravity_GetBlocklistUrls() { echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..." if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then - # Remove superceded $adListDefault file + # Remove superseded $adListDefault file rm "${adListDefault}" 2> /dev/null || \ echo -e " ${CROSS} Unable to remove ${adListDefault}" fi @@ -758,8 +744,6 @@ if [[ "${forceDelete:-}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" fi -detect_pihole_blocking_status - # Determine which functions to run if [[ "${skipDownload}" == false ]]; then # Gravity needs to download blocklists From e57d0fb93c24e6d7f336f449dc9bd1fc7627062d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 19:05:08 +0200 Subject: [PATCH 029/207] Remove the empty templates gravity.list, white.list, black.list. The old source files whitelist.txt, blacklist.txt, and regex.list still exist and need to be removed in a follow-up commit Signed-off-by: DL6ER --- gravity.sh | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/gravity.sh b/gravity.sh index b287653d..6313b12c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -32,10 +32,7 @@ whitelistFile="${piholeDir}/whitelist.txt" blacklistFile="${piholeDir}/blacklist.txt" regexFile="${piholeDir}/regex.list" -adList="${piholeDir}/gravity.list" -blackList="${piholeDir}/black.list" localList="${piholeDir}/local.list" -whiteList="${piholeDir}/white.list" VPNList="/etc/openvpn/ipp.txt" piholeGitDir="/etc/.pihole" @@ -146,18 +143,6 @@ gravity_store_in_database() { # Delete tmpfile rm "$tmpFile" > /dev/null 2>&1 || true - - # Only create template file if asked to - if [ "${template}" != "-" ]; then - # Empty $template if it already exists, otherwise, create it - output=$( { : > "${template}"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to create empty ${template}\\n ${output}" - gravity_Cleanup "error" - fi - fi } # Determine if DNS resolution is available before proceeding @@ -569,7 +554,7 @@ gravity_Whitelist() { echo -ne " ${INFO} ${str}..." # Store whitelisted files in gravity database - gravity_store_in_database "whitelist" "${whitelistFile}" "${whiteList}" + gravity_store_in_database "whitelist" "${whitelistFile}" echo -e "${OVER} ${INFO} ${str}" } @@ -589,7 +574,7 @@ gravity_ShowBlockCount() { fi # Store regex files in gravity database - gravity_store_in_database "regex" "${regexFile}" "-" + gravity_store_in_database "regex" "${regexFile}" } # Parse list of domains into hosts format @@ -639,7 +624,7 @@ gravity_ParseBlacklistDomains() { local output status # Store gravity domains in gravity database - gravity_store_in_database "gravity" "${piholeDir}/${preEventHorizon}" "${adList}" + gravity_store_in_database "gravity" "${piholeDir}/${preEventHorizon}" } # Create user-added blacklist entries @@ -649,7 +634,7 @@ gravity_ParseUserDomains() { fi # Fill database table - gravity_store_in_database "blacklist" "${blacklistFile}" "${blackList}" + gravity_store_in_database "blacklist" "${blacklistFile}" } # Trap Ctrl-C From 61cfd2f9f9ec4294e041937748f08368b80e88d5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 19:48:06 +0200 Subject: [PATCH 030/207] Add adlists table Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 3a2230c8..5442987f 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,6 +1,7 @@ CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); CREATE TABLE regex (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); +CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); From c1277705d897ce5ca15aa4d76b1ba253b42f7e6a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 24 Apr 2019 19:55:05 +0200 Subject: [PATCH 031/207] Copy existing whitelist.txt, blacklist.txt, regex.list, and adlists.list to the database. We remove the files afterwards as the content lives in the database now Signed-off-by: DL6ER --- gravity.sh | 62 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/gravity.sh b/gravity.sh index 6313b12c..4fb601b4 100755 --- a/gravity.sh +++ b/gravity.sh @@ -25,12 +25,11 @@ PIHOLE_GROUP="pihole" piholeDir="/etc/${basename}" -adListFile="${piholeDir}/adlists.list" -adListDefault="${piholeDir}/adlists.default" - +# Legacy (pre v5.0) list file locations whitelistFile="${piholeDir}/whitelist.txt" blacklistFile="${piholeDir}/blacklist.txt" regexFile="${piholeDir}/regex.list" +adListFile="${piholeDir}/adlists.list" localList="${piholeDir}/local.list" VPNList="/etc/openvpn/ipp.txt" @@ -93,18 +92,20 @@ generate_gravity_database() { } # Import domains from file and store them in the specified database table -gravity_store_in_database() { +database_table_from_file() { # Define locals local table="${1}" local source="${2}" - local template="${3}" # Create database file if not present if [ ! -e "${gravityDBfile}" ]; then + echo -e " ${INFO} Creating new gravity database" generate_gravity_database fi - # Empty domains + echo -e " ${INFO} Pi-hole upgrade: Moving content of ${source} into database" + + # Truncate table output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 ) status="$?" @@ -118,19 +119,19 @@ gravity_store_in_database() { local timestamp timestamp="$(date --utc +'%s')" local inputfile - if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then - # Apply format for white-, blacklist, and regex tables + if [[ "${table}" == "gravity" ]]; then + # No need to modify the input data for the gravity table + inputfile="${source}" + else + # Apply format for white-, blacklist, regex, and adlists tables # Read file line by line grep -v '^ *#' < "${source}" | while IFS= read -r domain do echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}" done inputfile="${tmpFile}" - else - # No need to modify the input data for the gravity table - inputfile="${source}" fi - # Store domains in gravity database table ${table} + # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 output=$( { printf ".mode csv\\n.import \"%s\" %s\\n" "${inputfile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) @@ -145,6 +146,31 @@ gravity_store_in_database() { rm "$tmpFile" > /dev/null 2>&1 || true } +migrate_to_database() { +ls -lh + # Migrate pre-v5.0 list files to database-based Pi-hole versions + if [[ -e "${whitelistFile}" ]]; then + # Store whitelisted domains in database + database_table_from_file "whitelist" "${whitelistFile}" + rm "${whitelistFile}" + fi + if [[ -e "${blacklistFile}" ]]; then + # Store blacklisted domains in database + database_table_from_file "blacklist" "${blacklistFile}" + rm "${blacklistFile}" + fi + if [[ -e "${regexFile}" ]]; then + # Store regex domains in database + database_table_from_file "regex" "${regexFile}" + rm "${regexFile}" + fi + if [[ -e "${adListFile}" ]]; then + # Store adlists domains in database + database_table_from_file "adlists" "${adListFile}" + rm "${adListFile}" + fi +} + # Determine if DNS resolution is available before proceeding gravity_CheckDNSResolutionAvailable() { local lookupDomain="pi.hole" @@ -553,9 +579,6 @@ gravity_Whitelist() { str="Number of whitelisted domains: ${num}" echo -ne " ${INFO} ${str}..." - # Store whitelisted files in gravity database - gravity_store_in_database "whitelist" "${whitelistFile}" - echo -e "${OVER} ${INFO} ${str}" } @@ -574,7 +597,7 @@ gravity_ShowBlockCount() { fi # Store regex files in gravity database - gravity_store_in_database "regex" "${regexFile}" + database_table_from_file "regex" "${regexFile}" } # Parse list of domains into hosts format @@ -624,7 +647,7 @@ gravity_ParseBlacklistDomains() { local output status # Store gravity domains in gravity database - gravity_store_in_database "gravity" "${piholeDir}/${preEventHorizon}" + database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}" } # Create user-added blacklist entries @@ -634,7 +657,7 @@ gravity_ParseUserDomains() { fi # Fill database table - gravity_store_in_database "blacklist" "${blacklistFile}" + database_table_from_file "blacklist" "${blacklistFile}" } # Trap Ctrl-C @@ -721,6 +744,9 @@ done # Trap Ctrl-C gravity_Trap +# Move possibly existing legacy files to the gravity database +migrate_to_database + if [[ "${forceDelete:-}" == true ]]; then str="Deleting existing list cache" echo -ne "${INFO} ${str}..." From 2664ac0efc6dab8720170d55b09c2ea50d857f18 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 10:18:37 +0200 Subject: [PATCH 032/207] Improve error reporting Signed-off-by: DL6ER --- gravity.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 4fb601b4..870027a1 100755 --- a/gravity.sh +++ b/gravity.sh @@ -143,7 +143,12 @@ database_table_from_file() { fi # Delete tmpfile - rm "$tmpFile" > /dev/null 2>&1 || true + rm "${tmpFile}" > /dev/null 2>&1 || \ + echo -e " ${CROSS} Unable to remove ${tmpFile}" + + # Delete source file + rm "${source}" 2> /dev/null || \ + echo -e " ${CROSS} Unable to remove ${source}" } migrate_to_database() { @@ -152,22 +157,18 @@ ls -lh if [[ -e "${whitelistFile}" ]]; then # Store whitelisted domains in database database_table_from_file "whitelist" "${whitelistFile}" - rm "${whitelistFile}" fi if [[ -e "${blacklistFile}" ]]; then # Store blacklisted domains in database database_table_from_file "blacklist" "${blacklistFile}" - rm "${blacklistFile}" fi if [[ -e "${regexFile}" ]]; then # Store regex domains in database database_table_from_file "regex" "${regexFile}" - rm "${regexFile}" fi if [[ -e "${adListFile}" ]]; then # Store adlists domains in database database_table_from_file "adlists" "${adListFile}" - rm "${adListFile}" fi } From 465a39427b1a0f34b83ede17a9dbfb000a19a3bb Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 10:46:18 +0200 Subject: [PATCH 033/207] Source adlists from gravity database Signed-off-by: DL6ER --- gravity.sh | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/gravity.sh b/gravity.sh index 870027a1..0333a25d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -151,9 +151,8 @@ database_table_from_file() { echo -e " ${CROSS} Unable to remove ${source}" } +# Migrate pre-v5.0 list files to database-based Pi-hole versions migrate_to_database() { -ls -lh - # Migrate pre-v5.0 list files to database-based Pi-hole versions if [[ -e "${whitelistFile}" ]]; then # Store whitelisted domains in database database_table_from_file "whitelist" "${whitelistFile}" @@ -232,15 +231,9 @@ gravity_CheckDNSResolutionAvailable() { gravity_GetBlocklistUrls() { echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..." - if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then - # Remove superseded $adListDefault file - rm "${adListDefault}" 2> /dev/null || \ - echo -e " ${CROSS} Unable to remove ${adListDefault}" - fi - - # Retrieve source URLs from $adListFile - # Logic: Remove comments and empty lines - mapfile -t sources <<< "$(grep -v -E "^(#|$)" "${adListFile}" 2> /dev/null)" + # Retrieve source URLs from gravity database + # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true) + mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM adlists WHERE enabled = 1;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( From d94bbfca7f728258ed3f0574b9eb9bf783114652 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 11:18:54 +0200 Subject: [PATCH 034/207] Finish migrating gravity.db to using our new gravity database. This removes the --skip-download, --blacklist-only, --whitelist-only, and --wildcard-only options as it is not necessary to run gravity when modifying them from now on (gravity really only builds the gravity database and the local.list file). Signed-off-by: DL6ER --- gravity.sh | 171 +++++++++++++++-------------------------------------- 1 file changed, 49 insertions(+), 122 deletions(-) diff --git a/gravity.sh b/gravity.sh index 0333a25d..09e7b19c 100755 --- a/gravity.sh +++ b/gravity.sh @@ -44,8 +44,6 @@ matterAndLight="${basename}.0.matterandlight.txt" parsedMatter="${basename}.1.parsedmatter.txt" preEventHorizon="list.preEventHorizon" -skipDownload="false" - resolver="pihole-FTL" haveSourceUrls=true @@ -103,8 +101,6 @@ database_table_from_file() { generate_gravity_database fi - echo -e " ${INFO} Pi-hole upgrade: Moving content of ${source} into database" - # Truncate table output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 ) status="$?" @@ -153,22 +149,26 @@ database_table_from_file() { # Migrate pre-v5.0 list files to database-based Pi-hole versions migrate_to_database() { - if [[ -e "${whitelistFile}" ]]; then - # Store whitelisted domains in database - database_table_from_file "whitelist" "${whitelistFile}" + if [[ -e "${adListFile}" ]]; then + # Store adlists domains in database + echo -e " ${INFO} Pi-hole upgrade: Moving content of ${adListFile} into database" + database_table_from_file "adlists" "${adListFile}" fi if [[ -e "${blacklistFile}" ]]; then # Store blacklisted domains in database + echo -e " ${INFO} Pi-hole upgrade: Moving content of ${blacklistFile} into database" database_table_from_file "blacklist" "${blacklistFile}" fi + if [[ -e "${whitelistFile}" ]]; then + # Store whitelisted domains in database + echo -e " ${INFO} Pi-hole upgrade: Moving content of ${whitelistFile} into database" + database_table_from_file "whitelist" "${whitelistFile}" + fi if [[ -e "${regexFile}" ]]; then # Store regex domains in database + echo -e " ${INFO} Pi-hole upgrade: Moving content of ${regexFile} into database" database_table_from_file "regex" "${regexFile}" fi - if [[ -e "${adListFile}" ]]; then - # Store adlists domains in database - database_table_from_file "adlists" "${adListFile}" - fi } # Determine if DNS resolution is available before proceeding @@ -282,11 +282,9 @@ gravity_SetDownloadOptions() { *) cmd_ext="";; esac - if [[ "${skipDownload}" == false ]]; then - echo -e " ${INFO} Target: ${domain} (${url##*/})" - gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" - echo "" - fi + echo -e " ${INFO} Target: ${domain} (${url##*/})" + gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" + echo "" done gravity_Blackbody=true } @@ -543,55 +541,35 @@ gravity_SortAndFilterConsolidatedList() { if [[ "${haveSourceUrls}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" fi - echo -e " ${INFO} Number of domains being pulled in by gravity: ${COL_BLUE}${num}${COL_NC}" + echo -e " ${INFO} Gravity pulled in ${COL_BLUE}${num}${COL_NC} domains" str="Removing duplicate domains" - if [[ "${haveSourceUrls}" == true ]]; then - echo -ne " ${INFO} ${str}..." - fi - + echo -ne " ${INFO} ${str}..." sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}" + echo -e "${OVER} ${TICK} ${str}" - if [[ "${haveSourceUrls}" == true ]]; then - echo -e "${OVER} ${TICK} ${str}" - # Format $preEventHorizon line total as currency - num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") - echo -e " ${INFO} Number of unique domains trapped in the Event Horizon: ${COL_BLUE}${num}${COL_NC}" - fi + # Format $preEventHorizon line total as currency + num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") + str="Storing ${COL_BLUE}${num}${COL_NC} unique blocking domains in database" + echo -ne " ${INFO} ${str}..." + database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}" + echo -e "${OVER} ${TICK} ${str}" } -# Whitelist user-defined domains -gravity_Whitelist() { - local num str - - if [[ ! -f "${whitelistFile}" ]]; then - echo -e " ${INFO} Nothing to whitelist!" - return 0 - fi - - num=$(wc -l < "${whitelistFile}") - str="Number of whitelisted domains: ${num}" - echo -ne " ${INFO} ${str}..." - - echo -e "${OVER} ${INFO} ${str}" +# Report number of entries in a table +gravity_Table_Count() { + local table="${1}" + local str="${2}" + local num + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table} WHERE enabled = 1;")" + echo -e " ${INFO} Number of ${str}: ${num}" } # Output count of blacklisted domains and regex filters -gravity_ShowBlockCount() { - local num - - if [[ -f "${blacklistFile}" ]]; then - num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")") - echo -e " ${INFO} Number of blacklisted domains: ${num}" - fi - - if [[ -f "${regexFile}" ]]; then - num=$(grep -cv "^#" "${regexFile}") - echo -e " ${INFO} Number of regex filters: ${num}" - fi - - # Store regex files in gravity database - database_table_from_file "regex" "${regexFile}" +gravity_ShowCount() { + gravity_Table_Count "blacklist" "blacklisted domains" + gravity_Table_Count "whitelist" "whitelisted domains" + gravity_Table_Count "regex" "regex filters" } # Parse list of domains into hosts format @@ -611,7 +589,7 @@ gravity_ParseDomainsIntoHosts() { } # Create "localhost" entries into hosts format -gravity_ParseLocalDomains() { +gravity_generateLocalList() { local hostname if [[ -s "/etc/hostname" ]]; then @@ -636,24 +614,6 @@ gravity_ParseLocalDomains() { fi } -# Create primary blacklist entries -gravity_ParseBlacklistDomains() { - local output status - - # Store gravity domains in gravity database - database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}" -} - -# Create user-added blacklist entries -gravity_ParseUserDomains() { - if [[ ! -f "${blacklistFile}" ]]; then - return 0 - fi - - # Fill database table - database_table_from_file "blacklist" "${blacklistFile}" -} - # Trap Ctrl-C gravity_Trap() { trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT @@ -689,7 +649,7 @@ gravity_Cleanup() { str="Optimizing domains database" echo -ne " ${INFO} ${str}..." # Run VACUUM command on database to optimize it - output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 ) + output=$( { sqlite3 "${gravityDBfile}" "VACUUM;"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then @@ -728,10 +688,6 @@ for var in "$@"; do "-f" | "--force" ) forceDelete=true;; "-o" | "--optimize" ) optimize_database=true;; "-h" | "--help" ) helpFunc;; - "-sd" | "--skip-download" ) skipDownload=true;; - "-b" | "--blacklist-only" ) listType="blacklist";; - "-w" | "--whitelist-only" ) listType="whitelist";; - "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";; esac done @@ -749,54 +705,25 @@ if [[ "${forceDelete:-}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" fi -# Determine which functions to run -if [[ "${skipDownload}" == false ]]; then - # Gravity needs to download blocklists - gravity_CheckDNSResolutionAvailable - gravity_GetBlocklistUrls - if [[ "${haveSourceUrls}" == true ]]; then - gravity_SetDownloadOptions - fi - gravity_ConsolidateDownloadedBlocklists - gravity_SortAndFilterConsolidatedList -else - # Gravity needs to modify Blacklist/Whitelist/Wildcards - echo -e " ${INFO} Using cached Event Horizon list..." - numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") - echo -e " ${INFO} ${COL_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon" +# Gravity downloads blocklists next +gravity_CheckDNSResolutionAvailable +gravity_GetBlocklistUrls +if [[ "${haveSourceUrls}" == true ]]; then +gravity_SetDownloadOptions fi +# Build preEventHorizon +gravity_ConsolidateDownloadedBlocklists +gravity_SortAndFilterConsolidatedList -# Perform when downloading blocklists, or modifying the whitelist -if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then - gravity_Whitelist -fi - -convert_wildcard_to_regex -gravity_ShowBlockCount - -# Perform when downloading blocklists, or modifying the white/blacklist (not wildcards) -if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then - str="Parsing domains" - echo -ne " ${INFO} ${str}..." - - gravity_ParseUserDomains - - # Perform when downloading blocklists - if [[ ! "${listType:-}" == "blacklist" ]]; then - gravity_ParseLocalDomains - gravity_ParseBlacklistDomains - fi - - echo -e "${OVER} ${TICK} ${str}" - - gravity_Cleanup -fi +# Create local.list +gravity_generateLocalList +gravity_ShowCount +gravity_Cleanup echo "" # Determine if DNS has been restarted by this instance of gravity if [[ -z "${dnsWasOffline:-}" ]]; then - # Use "force-reload" when restarting dnsmasq for everything but Wildcards - "${PIHOLE_COMMAND}" restartdns "${dnsRestartType:-force-reload}" + "${PIHOLE_COMMAND}" restartdns reload fi "${PIHOLE_COMMAND}" status From e5d1cb5a2ef11ab4642bd3b8f80cb2edcd9cb2b5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 12:10:42 +0200 Subject: [PATCH 035/207] Rewrite list.sh to operate on database-based lists Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 247 +++++++++++++++++---------------------- 1 file changed, 104 insertions(+), 143 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index c1d95aae..1a2094de 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -11,10 +11,8 @@ # Globals basename=pihole piholeDir=/etc/"${basename}" -whitelist="${piholeDir}"/whitelist.txt -blacklist="${piholeDir}"/blacklist.txt +gravityDBfile="${piholeDir}/gravity.db" -readonly regexlist="/etc/pihole/regex.list" reload=false addmode=true verbose=true @@ -22,35 +20,34 @@ wildcard=false domList=() -listMain="" -listAlt="" +listType="" colfile="/opt/pihole/COL_TABLE" source ${colfile} helpFunc() { - if [[ "${listMain}" == "${whitelist}" ]]; then + if [[ "${listType}" == "whitelist" ]]; then param="w" - type="white" - elif [[ "${listMain}" == "${regexlist}" && "${wildcard}" == true ]]; then + type="whitelist" + elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then param="-wild" - type="wildcard black" - elif [[ "${listMain}" == "${regexlist}" ]]; then + type="wildcard blacklist" + elif [[ "${listType}" == "regex" ]]; then param="-regex" - type="regex black" + type="regex filter" else param="b" - type="black" + type="blacklist" fi echo "Usage: pihole -${param} [options] Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com' -${type^}list one or more domains +${type^} one or more domains Options: - -d, --delmode Remove domain(s) from the ${type}list - -nr, --noreload Update ${type}list without refreshing dnsmasq + -d, --delmode Remove domain(s) from the ${type} + -nr, --noreload Update ${type} without reloading the DNS server -q, --quiet Make output less verbose -h, --help Show this help dialog -l, --list Display all your ${type}listed domains @@ -73,7 +70,7 @@ HandleOther() { # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then - if [[ "${listMain}" == "${regexlist}" && "${wildcard}" == false ]]; then + if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then validDomain="${domain}" else validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check @@ -88,169 +85,134 @@ HandleOther() { fi } -PoplistFile() { - # Check whitelist file exists, and if not, create it - if [[ ! -f "${whitelist}" ]]; then - touch "${whitelist}" - fi - - # Check blacklist file exists, and if not, create it - if [[ ! -f "${blacklist}" ]]; then - touch "${blacklist}" - fi - +ProcessDomainList() { for dom in "${domList[@]}"; do # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other if ${addmode}; then - AddDomain "${dom}" "${listMain}" + AddDomain "${dom}" "${listType}" RemoveDomain "${dom}" "${listAlt}" else - RemoveDomain "${dom}" "${listMain}" + RemoveDomain "${dom}" "${listType}" fi done } AddDomain() { + local domain list listname sqlitekey num + domain="$1" list="$2" - domain=$(EscapeRegexp "$1") - [[ "${list}" == "${whitelist}" ]] && listname="whitelist" - [[ "${list}" == "${blacklist}" ]] && listname="blacklist" - - if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then - [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only" - [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only" - bool=true - # Is the domain in the list we want to add it to? - grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false - - if [[ "${bool}" == false ]]; then - # Domain not found in the whitelist file, add it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${1} to ${listname}..." - fi - reload=true - # Add it to the list we want to add it to - echo "$1" >> "${list}" - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" - fi + if [[ "${list}" == "regex" ]]; then + listname="regex filters" + sqlitekey="filter" + if [[ "${wildcard}" == true ]]; then + domain="(^|\\.)${domain//\./\\.}$" fi - elif [[ "${list}" == "${regexlist}" ]]; then - [[ -z "${type}" ]] && type="--wildcard-only" - bool=true - domain="${1}" + else + # Whitelist / Blacklist + listname="${list}list" + sqlitekey="domain" + fi - [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$" + # Is the domain in the list we want to add it to? + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" - # Is the domain in the list? - # Search only for exactly matching lines - grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false - - if [[ "${bool}" == false ]]; then - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${domain} to regex list..." - fi - reload="restart" - echo "$domain" >> "${regexlist}" - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${domain} already exists in regex list, no need to add!" - fi + if [[ "${num}" -eq 0 ]]; then + # Domain not found in the file, add it! + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} Adding ${1} to ${listname}..." + fi + reload=true + # Add it to the list we want to add it to + local timestamp + timestamp="$(date --utc +'%s')" + sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (${sqlitekey},enabled,date_added) VALUES (\"${domain}\",1,${timestamp});" + else + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" fi fi } RemoveDomain() { + local domain list listname sqlitekey num + domain="$1" list="$2" - domain=$(EscapeRegexp "$1") - [[ "${list}" == "${whitelist}" ]] && listname="whitelist" - [[ "${list}" == "${blacklist}" ]] && listname="blacklist" - - if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then - bool=true - [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only" - [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only" - # Is it in the list? Logic follows that if its whitelisted it should not be blacklisted and vice versa - grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false - if [[ "${bool}" == true ]]; then - # Remove it from the other one - echo -e " ${INFO} Removing $1 from ${listname}..." - # /I flag: search case-insensitive - sed -i "/${domain}/Id" "${list}" - reload=true - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" - fi + if [[ "${list}" == "regex" ]]; then + listname="regex filters" + sqlitekey="filter" + if [[ "${wildcard}" == true ]]; then + domain="(^|\\.)${domain//\./\\.}$" fi - elif [[ "${list}" == "${regexlist}" ]]; then - [[ -z "${type}" ]] && type="--wildcard-only" - domain="${1}" + else + # Whitelist / Blacklist + listname="${list}list" + sqlitekey="domain" + fi - [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$" + # Is the domain in the list we want to remove it from? + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" - bool=true - # Is it in the list? - grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false - if [[ "${bool}" == true ]]; then - # Remove it from the other one - echo -e " ${INFO} Removing $domain from regex list..." - local lineNumber - lineNumber=$(grep -Fnx "$domain" "${list}" | cut -f1 -d:) - sed -i "${lineNumber}d" "${list}" - reload=true - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${domain} does not exist in regex list, no need to remove!" - fi + if [[ "${num}" -ne 0 ]]; then + # Domain found in the file, remove it! + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} Removing ${1} from ${listname}..." + fi + reload=true + # Remove it from the current list + local timestamp + timestamp="$(date --utc +'%s')" + sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";" + else + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" fi fi } -# Update Gravity -Reload() { - echo "" - pihole -g --skip-download "${type:-}" -} - Displaylist() { - if [[ -f ${listMain} ]]; then - if [[ "${listMain}" == "${whitelist}" ]]; then - string="gravity resistant domains" - else - string="domains caught in the sinkhole" - fi - verbose=false - echo -e "Displaying $string:\n" - count=1 - while IFS= read -r RD || [ -n "${RD}" ]; do - echo " ${count}: ${RD}" - count=$((count+1)) - done < "${listMain}" + local domain list listname count status + + if [[ "${listType}" == "regex" ]]; then + listname="regex filters list" else - echo -e " ${COL_LIGHT_RED}${listMain} does not exist!${COL_NC}" + # Whitelist / Blacklist + listname="${listType}" + fi + data="$(sqlite3 "${gravityDBfile}" "SELECT * FROM ${listType};" 2> /dev/null)" + + if [[ -z $data ]]; then + echo -e "Not showing empty ${listname}" + else + echo -e "Displaying ${listname}:" + count=1 + while IFS= read -r line + do + domain="$(cut -d'|' -f1 <<< "${line}")" + enabled="$(cut -d'|' -f2 <<< "${line}")" + if [[ "${enabled}" -eq 1 ]]; then + status="enabled" + else + status="disabled" + fi + echo " ${count}: ${domain} (${status})" + count=$((count+1)) + done <<< "${data}" + exit 0; fi - exit 0; } NukeList() { - if [[ -f "${listMain}" ]]; then - # Back up original list - cp "${listMain}" "${listMain}.bck~" - # Empty out file - echo "" > "${listMain}" - fi + sqlite3 "${gravityDBfile}" "DELETE FROM ${listType};" } for var in "$@"; do case "${var}" in - "-w" | "whitelist" ) listMain="${whitelist}"; listAlt="${blacklist}";; - "-b" | "blacklist" ) listMain="${blacklist}"; listAlt="${whitelist}";; - "--wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;; - "--regex" | "regex" ) listMain="${regexlist}";; + "-w" | "whitelist" ) listType="whitelist"; listAlt="blacklist";; + "-b" | "blacklist" ) listType="blacklist"; listAlt="whitelist";; + "--wild" | "wildcard" ) listType="regex"; wildcard=true;; + "--regex" | "regex" ) listType="regex";; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; @@ -267,9 +229,8 @@ if [[ $# = 0 ]]; then helpFunc fi -PoplistFile +ProcessDomainList if [[ "${reload}" != false ]]; then - # Ensure that "restart" is used for Wildcard updates - Reload "${reload}" + pihole restartdns reload fi From 788cd783212828229fb8708f38841b4a89472267 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 12:30:38 +0200 Subject: [PATCH 036/207] Reduce code duplication and add special handling of | character as it might appear in regex filter string Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 70 +++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 34 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 1a2094de..727919fe 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -21,9 +21,11 @@ wildcard=false domList=() listType="" +listname="" +sqlitekey="" -colfile="/opt/pihole/COL_TABLE" -source ${colfile} +# shellcheck source=/opt/pihole/COL_TABLE +source "/opt/pihole/COL_TABLE" helpFunc() { @@ -61,7 +63,7 @@ EscapeRegexp() { # string in our regular expressions # This sed is intentionally executed in three steps to ease maintainability # The first sed removes any amount of leading dots - echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g" + echo "$@" | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g" } HandleOther() { @@ -86,11 +88,28 @@ HandleOther() { } ProcessDomainList() { + if [[ "${listType}" == "regex" ]]; then + # Regex filter list + listname="regex filters" + sqlitekey="filter" + else + # Whitelist / Blacklist + listname="${listType}" + sqlitekey="domain" + fi + for dom in "${domList[@]}"; do - # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other + # Format domain into regex filter if requested + if [[ "${wildcard}" == true ]]; then + dom="(^|\\.)${dom//\./\\.}$" + fi + + # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the othe if ${addmode}; then AddDomain "${dom}" "${listType}" - RemoveDomain "${dom}" "${listAlt}" + if [[ ! "${listType}" == "regex" ]]; then + RemoveDomain "${dom}" "${listAlt}" + fi else RemoveDomain "${dom}" "${listType}" fi @@ -98,22 +117,10 @@ ProcessDomainList() { } AddDomain() { - local domain list listname sqlitekey num + local domain list num domain="$1" list="$2" - if [[ "${list}" == "regex" ]]; then - listname="regex filters" - sqlitekey="filter" - if [[ "${wildcard}" == true ]]; then - domain="(^|\\.)${domain//\./\\.}$" - fi - else - # Whitelist / Blacklist - listname="${list}list" - sqlitekey="domain" - fi - # Is the domain in the list we want to add it to? num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" @@ -135,22 +142,10 @@ AddDomain() { } RemoveDomain() { - local domain list listname sqlitekey num + local domain list num domain="$1" list="$2" - if [[ "${list}" == "regex" ]]; then - listname="regex filters" - sqlitekey="filter" - if [[ "${wildcard}" == true ]]; then - domain="(^|\\.)${domain//\./\\.}$" - fi - else - # Whitelist / Blacklist - listname="${list}list" - sqlitekey="domain" - fi - # Is the domain in the list we want to remove it from? num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" @@ -172,7 +167,7 @@ RemoveDomain() { } Displaylist() { - local domain list listname count status + local list listname count num_pipes domain enabled status if [[ "${listType}" == "regex" ]]; then listname="regex filters list" @@ -189,13 +184,20 @@ Displaylist() { count=1 while IFS= read -r line do - domain="$(cut -d'|' -f1 <<< "${line}")" - enabled="$(cut -d'|' -f2 <<< "${line}")" + # Count number of pipes seen in this line + # This is necessary because we can only detect the pipe separating the fields + # from the end backwards as the domain (which is the first field) may contain + # pipe symbols as they are perfectly valid regex filter control characters + num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")" + + domain="$(cut -d'|' -f"-$((num_pipes-2))" <<< "${line}")" + enabled="$(cut -d'|' -f$((num_pipes-1)) <<< "${line}")" if [[ "${enabled}" -eq 1 ]]; then status="enabled" else status="disabled" fi + echo " ${count}: ${domain} (${status})" count=$((count+1)) done <<< "${data}" From 43bced7997772d5604d1302e1c8fbc6369ec45e1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 12:45:08 +0200 Subject: [PATCH 037/207] Add more comments Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 727919fe..5fd24f6b 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -24,8 +24,8 @@ listType="" listname="" sqlitekey="" -# shellcheck source=/opt/pihole/COL_TABLE -source "/opt/pihole/COL_TABLE" +colfile="/opt/pihole/COL_TABLE" +source ${colfile} helpFunc() { @@ -63,7 +63,7 @@ EscapeRegexp() { # string in our regular expressions # This sed is intentionally executed in three steps to ease maintainability # The first sed removes any amount of leading dots - echo "$@" | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g" + echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g" } HandleOther() { @@ -190,8 +190,11 @@ Displaylist() { # pipe symbols as they are perfectly valid regex filter control characters num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")" + # Extract domain and enabled status based on the obtained number of pipe characters domain="$(cut -d'|' -f"-$((num_pipes-2))" <<< "${line}")" - enabled="$(cut -d'|' -f$((num_pipes-1)) <<< "${line}")" + enabled="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")" + + # Translate boolean status into human readable string if [[ "${enabled}" -eq 1 ]]; then status="enabled" else From cce66a13b69ece62037b9dd2a01cf4aab18a3abe Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 14:54:41 +0200 Subject: [PATCH 038/207] Fix output in list.sh when adding a domain that is already present in the target list Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 5fd24f6b..1ad99468 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -161,7 +161,7 @@ RemoveDomain() { sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";" else if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" + echo -e " ${INFO} ${1} does not exist in ${list}, no need to remove!" fi fi } From a2a7e8000795d2b58071d7393b86cf2bc3e894fc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 25 Apr 2019 15:01:13 +0200 Subject: [PATCH 039/207] Add --web for sending a confirmation string when executed from the web interface. This is necessary as the "Reloading DNS service" message does not always propagate through to the web interface when reloading pihole-FTL takes a while. Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 1ad99468..969e7e44 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -17,6 +17,7 @@ reload=false addmode=true verbose=true wildcard=false +web=false domList=() @@ -224,6 +225,7 @@ for var in "$@"; do "-h" | "--help" ) helpFunc;; "-l" | "--list" ) Displaylist;; "--nuke" ) NukeList;; + "--web" ) web=true;; * ) HandleOther "${var}";; esac done @@ -236,6 +238,11 @@ fi ProcessDomainList +# Used on web interface +if $web; then +echo "DONE" +fi + if [[ "${reload}" != false ]]; then pihole restartdns reload fi From 0efc46260f0726edf0e2986df536c0f4e0c93fdc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 16:16:30 +0200 Subject: [PATCH 040/207] Use both old and new strings for detecting whether this is a Pi-hole dnsmasq config file Signed-off-by: DL6ER --- automated install/basic-install.sh | 6 ++++-- gravity.sh | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 6bcfb564..06facdf7 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1224,7 +1224,8 @@ version_check_dnsmasq() { # Local, named variables local dnsmasq_conf="/etc/dnsmasq.conf" local dnsmasq_conf_orig="/etc/dnsmasq.conf.orig" - local dnsmasq_pihole_id_string="# Dnsmasq config for Pi-hole's FTLDNS" + local dnsmasq_pihole_id_string="addn-hosts=/etc/pihole/gravity.list" + local dnsmasq_pihole_id_string2="# Dnsmasq config for Pi-hole's FTLDNS" local dnsmasq_original_config="${PI_HOLE_LOCAL_REPO}/advanced/dnsmasq.conf.original" local dnsmasq_pihole_01_snippet="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf" local dnsmasq_pihole_01_location="/etc/dnsmasq.d/01-pihole.conf" @@ -1233,7 +1234,8 @@ version_check_dnsmasq() { if [[ -f "${dnsmasq_conf}" ]]; then printf " %b Existing dnsmasq.conf found..." "${INFO}" # If a specific string is found within this file, we presume it's from older versions on Pi-hole, - if grep -q ${dnsmasq_pihole_id_string} ${dnsmasq_conf}; then + if grep -q "${dnsmasq_pihole_id_string}" "${dnsmasq_conf}" || + grep -q "${dnsmasq_pihole_id_string2}" "${dnsmasq_conf}"; then printf " it is from a previous Pi-hole install.\\n" printf " %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}" # so backup the original file diff --git a/gravity.sh b/gravity.sh index 09e7b19c..75057eec 100755 --- a/gravity.sh +++ b/gravity.sh @@ -20,8 +20,9 @@ source "${regexconverter}" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" -PIHOLE_USER="pihole" -PIHOLE_GROUP="pihole" +DATABASE_USER="pihole" +DATABASE_GROUP="www-data" +DATABASE_PERMISSIONS="0660" piholeDir="/etc/${basename}" @@ -86,7 +87,8 @@ fi # Generate new sqlite3 file from schema template generate_gravity_database() { sqlite3 "${gravityDBfile}" < "${gravityDBschema}" - chown $PIHOLE_USER:$PIHOLE_GROUP "${gravityDBfile}" + chown ${DATABASE_USER}:${DATABASE_GROUP} "${piholeDir}" "${gravityDBfile}" + chmod ${DATABASE_PERMISSIONS} "${piholeDir}" "${gravityDBfile}" } # Import domains from file and store them in the specified database table From 08ee9526adde9fa3bd18cffa8b58ef148c2ccc52 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:06:01 +0200 Subject: [PATCH 041/207] Review comments Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 76 +++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 969e7e44..af44013e 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -105,7 +105,8 @@ ProcessDomainList() { dom="(^|\\.)${dom//\./\\.}$" fi - # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the othe + # Logic: If addmode then add to desired list and remove from the other; + # if delmode then remove from desired list but do not add to the other if ${addmode}; then AddDomain "${dom}" "${listType}" if [[ ! "${listType}" == "regex" ]]; then @@ -119,63 +120,64 @@ ProcessDomainList() { AddDomain() { local domain list num - domain="$1" + # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input + domain="$(printf "%q" "$1")" list="$2" # Is the domain in the list we want to add it to? num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" - if [[ "${num}" -eq 0 ]]; then - # Domain not found in the file, add it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${1} to ${listname}..." - fi - reload=true - # Add it to the list we want to add it to - local timestamp - timestamp="$(date --utc +'%s')" - sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (${sqlitekey},enabled,date_added) VALUES (\"${domain}\",1,${timestamp});" - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" - fi + if [[ "${num}" -ne 0 ]]; then + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" + fi + return fi + + # Domain not found in the table, add it! + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} Adding ${1} to ${listname}..." + fi + reload=true + # Add it to the list we want to add it to + local timestamp + timestamp="$(date --utc +'%s')" + # Insert only domain and date_added here. The enabled fields will be filled + # with its default value is true. + sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (${sqlitekey},date_added) VALUES (\"${domain}\",${timestamp});" } RemoveDomain() { local domain list num - domain="$1" + # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input + domain="$(printf "%q" "$1")" list="$2" # Is the domain in the list we want to remove it from? num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" - if [[ "${num}" -ne 0 ]]; then - # Domain found in the file, remove it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Removing ${1} from ${listname}..." - fi - reload=true - # Remove it from the current list - local timestamp - timestamp="$(date --utc +'%s')" - sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";" - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} does not exist in ${list}, no need to remove!" - fi + if [[ "${num}" -eq 0 ]]; then + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} does not exist in ${list}, no need to remove!" + fi + return fi + + # Domain found in the table, remove it! + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} Removing ${1} from ${listname}..." + fi + reload=true + # Remove it from the current list + local timestamp + timestamp="$(date --utc +'%s')" + sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";" } Displaylist() { local list listname count num_pipes domain enabled status - if [[ "${listType}" == "regex" ]]; then - listname="regex filters list" - else - # Whitelist / Blacklist - listname="${listType}" - fi + listname="${listType}" data="$(sqlite3 "${gravityDBfile}" "SELECT * FROM ${listType};" 2> /dev/null)" if [[ -z $data ]]; then From 6b1d2523e8bea9310e5095208f0d4732cef14cb9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:07:16 +0200 Subject: [PATCH 042/207] Also print datetime a domain was added when listing the domains on the CLI Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index af44013e..d11ab96c 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -175,7 +175,7 @@ RemoveDomain() { } Displaylist() { - local list listname count num_pipes domain enabled status + local list listname count num_pipes domain enabled status nicedate listname="${listType}" data="$(sqlite3 "${gravityDBfile}" "SELECT * FROM ${listType};" 2> /dev/null)" @@ -196,7 +196,9 @@ Displaylist() { # Extract domain and enabled status based on the obtained number of pipe characters domain="$(cut -d'|' -f"-$((num_pipes-2))" <<< "${line}")" enabled="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")" + dateadded="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")" + echo "${dateadded}" # Translate boolean status into human readable string if [[ "${enabled}" -eq 1 ]]; then status="enabled" @@ -204,7 +206,10 @@ Displaylist() { status="disabled" fi - echo " ${count}: ${domain} (${status})" + # Get nice representation of numerical date stored in database + nicedate=$(date --rfc-2822 -d "@${dateadded}") + + echo " ${count}: ${domain} (${status}, added ${nicedate})" count=$((count+1)) done <<< "${data}" exit 0; From a891d6439533f64a42baacdb409df4725ac7461e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:13:36 +0200 Subject: [PATCH 043/207] We should always exit after displaying a list, even if it is empty Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index d11ab96c..cd9dce52 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -212,8 +212,8 @@ Displaylist() { echo " ${count}: ${domain} (${status}, added ${nicedate})" count=$((count+1)) done <<< "${data}" - exit 0; fi + exit 0; } NukeList() { From 5bb816367422c5c6394543ff88ed8b22f31cf60f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:17:21 +0200 Subject: [PATCH 044/207] Change type of whitelist, blacklist, regex, and adlists time column to INTEGER and insert a DEFAULT clause that returns the current Unix time as such an integer Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 5442987f..cd62c94e 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,7 +1,7 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); -CREATE TABLE regex (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); -CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT); +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); From 53e3ff2b2421a491badb24be578717e3bc11e485 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:32:05 +0200 Subject: [PATCH 045/207] Only add non-empty lines when moving file content into database tables Signed-off-by: DL6ER --- gravity.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 75057eec..1d94424f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -125,7 +125,10 @@ database_table_from_file() { # Read file line by line grep -v '^ *#' < "${source}" | while IFS= read -r domain do - echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}" + # Only add non-empty lines + if [[ ! -z "${domain}" ]]; then + echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}" + fi done inputfile="${tmpFile}" fi From bd0215524ba0191b5364a596e769027a22be88fd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:36:00 +0200 Subject: [PATCH 046/207] Add vw_adlists view which returns only enabled adlists Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 6 +++++- gravity.sh | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index cd62c94e..ebe97955 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -20,6 +20,10 @@ CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain FROM whitelist a WHERE a.enabled == 1; -CREATE VIEW vw_regex AS SELECT DISTINCT a.filter +CREATE VIEW vw_regex AS SELECT DISTINCT a.domain FROM regex a WHERE a.enabled == 1; + +CREATE VIEW vw_adlists AS SELECT DISTINCT a.address +FROM adlists a +WHERE a.enabled == 1; diff --git a/gravity.sh b/gravity.sh index 1d94424f..a0652ced 100755 --- a/gravity.sh +++ b/gravity.sh @@ -238,7 +238,7 @@ gravity_GetBlocklistUrls() { # Retrieve source URLs from gravity database # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM adlists WHERE enabled = 1;" 2> /dev/null)" + mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( From 3dcb9722d35fb42d8a63903d5808a493e0b8507c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:39:35 +0200 Subject: [PATCH 047/207] Remove global haveSourceUrls variable and just skip downloading and processing adlists if there aren't any to be downloaded Signed-off-by: DL6ER --- gravity.sh | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/gravity.sh b/gravity.sh index a0652ced..3543ac15 100755 --- a/gravity.sh +++ b/gravity.sh @@ -47,8 +47,6 @@ preEventHorizon="list.preEventHorizon" resolver="pihole-FTL" -haveSourceUrls=true - # Source setupVars from install script setupVars="${piholeDir}/setupVars.conf" if [[ -f "${setupVars}" ]];then @@ -255,11 +253,12 @@ gravity_GetBlocklistUrls() { if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then echo -e "${OVER} ${TICK} ${str}" + return 0 else echo -e "${OVER} ${CROSS} ${str}" echo -e " ${INFO} No source list found, or it is empty" echo "" - haveSourceUrls=false + return 1 fi } @@ -503,9 +502,7 @@ gravity_ConsolidateDownloadedBlocklists() { local str lastLine str="Consolidating blocklists" - if [[ "${haveSourceUrls}" == true ]]; then - echo -ne " ${INFO} ${str}..." - fi + echo -ne " ${INFO} ${str}..." # Empty $matterAndLight if it already exists, otherwise, create it : > "${piholeDir}/${matterAndLight}" @@ -524,9 +521,8 @@ gravity_ConsolidateDownloadedBlocklists() { fi fi done - if [[ "${haveSourceUrls}" == true ]]; then - echo -e "${OVER} ${TICK} ${str}" - fi + echo -e "${OVER} ${TICK} ${str}" + } # Parse consolidated list into (filtered, unique) domains-only format @@ -534,18 +530,15 @@ gravity_SortAndFilterConsolidatedList() { local str num str="Extracting domains from blocklists" - if [[ "${haveSourceUrls}" == true ]]; then - echo -ne " ${INFO} ${str}..." - fi + echo -ne " ${INFO} ${str}..." # Parse into file gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}" # Format $parsedMatter line total as currency num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")") - if [[ "${haveSourceUrls}" == true ]]; then - echo -e "${OVER} ${TICK} ${str}" - fi + + echo -e "${OVER} ${TICK} ${str}" echo -e " ${INFO} Gravity pulled in ${COL_BLUE}${num}${COL_NC} domains" str="Removing duplicate domains" @@ -713,12 +706,12 @@ fi # Gravity downloads blocklists next gravity_CheckDNSResolutionAvailable gravity_GetBlocklistUrls -if [[ "${haveSourceUrls}" == true ]]; then -gravity_SetDownloadOptions +if gravity_GetBlocklistUrls; then + gravity_SetDownloadOptions + # Build preEventHorizon + gravity_ConsolidateDownloadedBlocklists + gravity_SortAndFilterConsolidatedList fi -# Build preEventHorizon -gravity_ConsolidateDownloadedBlocklists -gravity_SortAndFilterConsolidatedList # Create local.list gravity_generateLocalList From e04dc9d2bb20400912067d1ff39c6dcc2873f76c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:44:42 +0200 Subject: [PATCH 048/207] Remove sqlitekey variable as the first column of the regex table is now called domain as well Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index cd9dce52..dcd6f580 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -23,7 +23,6 @@ domList=() listType="" listname="" -sqlitekey="" colfile="/opt/pihole/COL_TABLE" source ${colfile} @@ -92,11 +91,9 @@ ProcessDomainList() { if [[ "${listType}" == "regex" ]]; then # Regex filter list listname="regex filters" - sqlitekey="filter" else # Whitelist / Blacklist listname="${listType}" - sqlitekey="domain" fi for dom in "${domList[@]}"; do @@ -125,7 +122,7 @@ AddDomain() { list="$2" # Is the domain in the list we want to add it to? - num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = \"${domain}\";")" if [[ "${num}" -ne 0 ]]; then if [[ "${verbose}" == true ]]; then @@ -144,7 +141,7 @@ AddDomain() { timestamp="$(date --utc +'%s')" # Insert only domain and date_added here. The enabled fields will be filled # with its default value is true. - sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (${sqlitekey},date_added) VALUES (\"${domain}\",${timestamp});" + sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain,date_added) VALUES (\"${domain}\",${timestamp});" } RemoveDomain() { @@ -154,7 +151,7 @@ RemoveDomain() { list="$2" # Is the domain in the list we want to remove it from? - num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")" + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = \"${domain}\";")" if [[ "${num}" -eq 0 ]]; then if [[ "${verbose}" == true ]]; then @@ -171,7 +168,7 @@ RemoveDomain() { # Remove it from the current list local timestamp timestamp="$(date --utc +'%s')" - sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";" + sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = \"${domain}\";" } Displaylist() { From 8524aecfed5eb4e3984fe938d2049c47976741c7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 17:56:11 +0200 Subject: [PATCH 049/207] Don't set gravity database permissions Signed-off-by: DL6ER --- gravity.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/gravity.sh b/gravity.sh index 3543ac15..98db04b8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -20,9 +20,6 @@ source "${regexconverter}" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" -DATABASE_USER="pihole" -DATABASE_GROUP="www-data" -DATABASE_PERMISSIONS="0660" piholeDir="/etc/${basename}" @@ -85,8 +82,6 @@ fi # Generate new sqlite3 file from schema template generate_gravity_database() { sqlite3 "${gravityDBfile}" < "${gravityDBschema}" - chown ${DATABASE_USER}:${DATABASE_GROUP} "${piholeDir}" "${gravityDBfile}" - chmod ${DATABASE_PERMISSIONS} "${piholeDir}" "${gravityDBfile}" } # Import domains from file and store them in the specified database table From 3fe43ce1d923f825b73a4f9c367b3a7f1bf344c9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 18:15:53 +0200 Subject: [PATCH 050/207] Rely on default value (database provided) for date_added when adding a new domain Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index dcd6f580..053def02 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -133,15 +133,12 @@ AddDomain() { # Domain not found in the table, add it! if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${1} to ${listname}..." + echo -e " ${INFO} Adding ${1} to the ${listname}..." fi reload=true - # Add it to the list we want to add it to - local timestamp - timestamp="$(date --utc +'%s')" - # Insert only domain and date_added here. The enabled fields will be filled - # with its default value is true. - sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain,date_added) VALUES (\"${domain}\",${timestamp});" + # Insert only the domain here. The enabled and date_added fields will be filled + # with their default values (enabled = true, date_added = current timestamp) + sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES (\"${domain}\");" } RemoveDomain() { @@ -162,12 +159,10 @@ RemoveDomain() { # Domain found in the table, remove it! if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Removing ${1} from ${listname}..." + echo -e " ${INFO} Removing ${1} from the ${listname}..." fi reload=true # Remove it from the current list - local timestamp - timestamp="$(date --utc +'%s')" sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = \"${domain}\";" } @@ -195,7 +190,6 @@ Displaylist() { enabled="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")" dateadded="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")" - echo "${dateadded}" # Translate boolean status into human readable string if [[ "${enabled}" -eq 1 ]]; then status="enabled" From b4ae142149f60150d583ec49876dfdef7dd3d259 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 26 Apr 2019 20:54:01 +0200 Subject: [PATCH 051/207] Remove printf escaping (we will realize it differently) and ensure we're using single quotes for strings (although double quotes are possible, too) Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 053def02..374f888b 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -118,11 +118,11 @@ ProcessDomainList() { AddDomain() { local domain list num # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input - domain="$(printf "%q" "$1")" + domain="$1" list="$2" # Is the domain in the list we want to add it to? - num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = \"${domain}\";")" + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")" if [[ "${num}" -ne 0 ]]; then if [[ "${verbose}" == true ]]; then @@ -138,17 +138,17 @@ AddDomain() { reload=true # Insert only the domain here. The enabled and date_added fields will be filled # with their default values (enabled = true, date_added = current timestamp) - sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES (\"${domain}\");" + sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES ('${domain}');" } RemoveDomain() { local domain list num # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input - domain="$(printf "%q" "$1")" + domain="$1" list="$2" # Is the domain in the list we want to remove it from? - num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = \"${domain}\";")" + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")" if [[ "${num}" -eq 0 ]]; then if [[ "${verbose}" == true ]]; then @@ -163,7 +163,7 @@ RemoveDomain() { fi reload=true # Remove it from the current list - sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = \"${domain}\";" + sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = '${domain}';" } Displaylist() { From 126f7101ffaa56b42891d9db1bed12e270222bd0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 28 Apr 2019 14:42:52 +0200 Subject: [PATCH 052/207] Add (optional) date_modified field and use vw_whitelist view in vw_blacklist Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 6 +++--- advanced/Templates/gravity.db.sql | 17 ++++++++--------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 374f888b..73cb2878 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -186,9 +186,9 @@ Displaylist() { num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")" # Extract domain and enabled status based on the obtained number of pipe characters - domain="$(cut -d'|' -f"-$((num_pipes-2))" <<< "${line}")" - enabled="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")" - dateadded="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")" + domain="$(cut -d'|' -f"-$((num_pipes-3))" <<< "${line}")" + enabled="$(cut -d'|' -f"$((num_pipes-2))" <<< "${line}")" + dateadded="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")" # Translate boolean status into human readable string if [[ "${enabled}" -eq 1 ]]; then diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index ebe97955..31fbfd9f 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,7 +1,7 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); +CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); +CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); @@ -11,15 +11,14 @@ CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain FROM gravity a WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); -CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain -FROM blacklist a -WHERE a.enabled == 1 AND - a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); - CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain FROM whitelist a WHERE a.enabled == 1; +CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain +FROM blacklist a +WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist; + CREATE VIEW vw_regex AS SELECT DISTINCT a.domain FROM regex a WHERE a.enabled == 1; From efeba594ae791000df0461b00e82e02251892038 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 28 Apr 2019 21:39:06 +0200 Subject: [PATCH 053/207] Add support for manipulating adlists from the CLI Signed-off-by: DL6ER --- advanced/Scripts/webpage.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index c6852896..6ac351bc 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -17,6 +17,8 @@ readonly FTLconf="/etc/pihole/pihole-FTL.conf" # 03 -> wildcards readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf" +readonly gravityDBfile="/etc/pihole/gravity.db" + coltable="/opt/pihole/COL_TABLE" if [[ -f ${coltable} ]]; then source ${coltable} @@ -385,19 +387,17 @@ SetWebUILayout() { } CustomizeAdLists() { - list="/etc/pihole/adlists.list" + local address + address="${args[3]}" if [[ "${args[2]}" == "enable" ]]; then - sed -i "\\@${args[3]}@s/^#http/http/g" "${list}" + sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 1 WHERE address = '${address}'" elif [[ "${args[2]}" == "disable" ]]; then - sed -i "\\@${args[3]}@s/^http/#http/g" "${list}" + sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 0 WHERE address = '${address}'" elif [[ "${args[2]}" == "add" ]]; then - if [[ $(grep -c "^${args[3]}$" "${list}") -eq 0 ]] ; then - echo "${args[3]}" >> ${list} - fi + sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlists (address) VALUES ('${address}')" elif [[ "${args[2]}" == "del" ]]; then - var=$(echo "${args[3]}" | sed 's/\//\\\//g') - sed -i "/${var}/Id" "${list}" + sqlite3 "${gravityDBfile}" "DELETE FROM adlists WHERE address = '${address}'" else echo "Not permitted" return 1 From 4946b5907bf4d45fda807558e2e0234ed70db784 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 28 Apr 2019 22:15:47 +0200 Subject: [PATCH 054/207] Move migrated files to backup directory instead of deleting them. This has the benefit that gravity.db can be recreated at any time by: 1. deleting gravity.db, 2. copying all files from the migration_backup directory back to /etc/pihole, 3. running pihole -g again. Signed-off-by: DL6ER --- gravity.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index a3456fb4..6e238e68 100755 --- a/gravity.sh +++ b/gravity.sh @@ -89,6 +89,8 @@ database_table_from_file() { # Define locals local table="${1}" local source="${2}" + local backup_path="${piholeDir}/migration_backup" + local backup_file="${backup_path}/$(basename "${2}")" # Create database file if not present if [ ! -e "${gravityDBfile}" ]; then @@ -140,8 +142,9 @@ database_table_from_file() { rm "${tmpFile}" > /dev/null 2>&1 || \ echo -e " ${CROSS} Unable to remove ${tmpFile}" - # Delete source file - rm "${source}" 2> /dev/null || \ + # Move source file to backup directory, create directory if not existing + mkdir -p "${backup_path}" + mv "${source}" "${backup_file}" 2> /dev/null || \ echo -e " ${CROSS} Unable to remove ${source}" } @@ -702,7 +705,6 @@ fi # Gravity downloads blocklists next gravity_CheckDNSResolutionAvailable -gravity_GetBlocklistUrls if gravity_GetBlocklistUrls; then gravity_SetDownloadOptions # Build preEventHorizon From a932209143af5eb9e3acd4991a5bb203375ba461 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 29 Apr 2019 17:19:02 +0200 Subject: [PATCH 055/207] Set default for date_modified to be the current time Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 8 ++++---- gravity.sh | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 31fbfd9f..c1066f33 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,7 +1,7 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); -CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); -CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT); +CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); diff --git a/gravity.sh b/gravity.sh index 6e238e68..331b632d 100755 --- a/gravity.sh +++ b/gravity.sh @@ -122,7 +122,7 @@ database_table_from_file() { do # Only add non-empty lines if [[ ! -z "${domain}" ]]; then - echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}" + echo "\"${domain}\",1,${timestamp},${timestamp}" >> "${tmpFile}" fi done inputfile="${tmpFile}" @@ -145,7 +145,7 @@ database_table_from_file() { # Move source file to backup directory, create directory if not existing mkdir -p "${backup_path}" mv "${source}" "${backup_file}" 2> /dev/null || \ - echo -e " ${CROSS} Unable to remove ${source}" + echo -e " ${CROSS} Unable to backup ${source} to ${backup_path}" } # Migrate pre-v5.0 list files to database-based Pi-hole versions From 2180531a84516f6720985f034fc8d0cdb589e6a7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 29 Apr 2019 19:43:35 +0200 Subject: [PATCH 056/207] Add trigger for automatic updates of the date_modified column and improve formating in the gravity.db template Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 37 ++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index c1066f33..dc0f23ed 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -8,21 +8,38 @@ CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); INSERT INTO info VALUES("version","1"); CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain -FROM gravity a -WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); + FROM gravity a + WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain -FROM whitelist a -WHERE a.enabled == 1; + FROM whitelist a + WHERE a.enabled == 1; +CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist + BEGIN + UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain -FROM blacklist a -WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist; + FROM blacklist a + WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist; +CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist + BEGIN + UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; CREATE VIEW vw_regex AS SELECT DISTINCT a.domain -FROM regex a -WHERE a.enabled == 1; + FROM regex a + WHERE a.enabled == 1; +CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex + BEGIN + UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; CREATE VIEW vw_adlists AS SELECT DISTINCT a.address -FROM adlists a -WHERE a.enabled == 1; + FROM adlists a + WHERE a.enabled == 1; +CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists + BEGIN + UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; + END; + From 5c575e73c7e3d9efd84a86740d7c09b0316089ff Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Tue, 30 Apr 2019 22:41:12 +0200 Subject: [PATCH 057/207] Adds world readable attribute to files created by Pi-hole to circumvent #2724 Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- advanced/Scripts/list.sh | 5 ++++- advanced/Scripts/piholeCheckout.sh | 3 +++ advanced/Scripts/piholeLogFlush.sh | 4 +++- advanced/Scripts/updatecheck.sh | 3 +++ advanced/Scripts/webpage.sh | 3 +++ automated install/basic-install.sh | 35 +++++++++++++++++++++++++----- automated install/uninstall.sh | 2 +- gravity.sh | 8 ++++++- pihole | 4 ++++ 9 files changed, 58 insertions(+), 9 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index c1d95aae..e0455e0c 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -92,11 +92,13 @@ PoplistFile() { # Check whitelist file exists, and if not, create it if [[ ! -f "${whitelist}" ]]; then touch "${whitelist}" + chmod a+r "${whitelist}" fi # Check blacklist file exists, and if not, create it if [[ ! -f "${blacklist}" ]]; then touch "${blacklist}" + chmod a+r "${blacklist}" fi for dom in "${domList[@]}"; do @@ -239,9 +241,10 @@ Displaylist() { NukeList() { if [[ -f "${listMain}" ]]; then # Back up original list - cp "${listMain}" "${listMain}.bck~" + cp -p "${listMain}" "${listMain}.bck~" # Empty out file echo "" > "${listMain}" + chmod a+r "${listMain}" fi } diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 1bfe5e21..a982289f 100644 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -90,6 +90,7 @@ checkout() { local path path="development/${binary}" echo "development" > /etc/pihole/ftlbranch + chmod a+r /etc/pihole/ftlbranch elif [[ "${1}" == "master" ]] ; then # Shortcut to check out master branches echo -e " ${INFO} Shortcut \"master\" detected - checking out master branches..." @@ -104,6 +105,7 @@ checkout() { local path path="master/${binary}" echo "master" > /etc/pihole/ftlbranch + chmod a+r /etc/pihole/ftlbranch elif [[ "${1}" == "core" ]] ; then str="Fetching branches from ${piholeGitUrl}" echo -ne " ${INFO} $str" @@ -166,6 +168,7 @@ checkout() { if check_download_exists "$path"; then echo " ${TICK} Branch ${2} exists" echo "${2}" > /etc/pihole/ftlbranch + chmod a+r /etc/pihole/ftlbranch FTLinstall "${binary}" restart_service pihole-FTL enable_service pihole-FTL diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 561fbce7..1e4b7abe 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -39,8 +39,9 @@ if [[ "$@" == *"once"* ]]; then # Note that moving the file is not an option, as # dnsmasq would happily continue writing into the # moved file (it will have the same file handler) - cp /var/log/pihole.log /var/log/pihole.log.1 + cp -p /var/log/pihole.log /var/log/pihole.log.1 echo " " > /var/log/pihole.log + chmod a+r /var/log/pihole.log fi else # Manual flushing @@ -53,6 +54,7 @@ else echo " " > /var/log/pihole.log if [ -f /var/log/pihole.log.1 ]; then echo " " > /var/log/pihole.log.1 + chmod a+r /var/log/pihole.log.1 fi fi # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 257c1929..55fce328 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -51,6 +51,7 @@ if [[ "$2" == "remote" ]]; then GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")" echo -n "${GITHUB_CORE_VERSION}" > "${GITHUB_VERSION_FILE}" + chmod a+r "${GITHUB_VERSION_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")" @@ -66,6 +67,7 @@ else CORE_BRANCH="$(get_local_branch /etc/.pihole)" echo -n "${CORE_BRANCH}" > "${LOCAL_BRANCH_FILE}" + chmod a+r "${LOCAL_BRANCH_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then WEB_BRANCH="$(get_local_branch /var/www/html/admin)" @@ -79,6 +81,7 @@ else CORE_VERSION="$(get_local_version /etc/.pihole)" echo -n "${CORE_VERSION}" > "${LOCAL_VERSION_FILE}" + chmod a+r "${LOCAL_VERSION_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then WEB_VERSION="$(get_local_version /var/www/html/admin)" diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index c6852896..eba539a6 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -322,6 +322,7 @@ dhcp-option=option:router,${DHCP_ROUTER} dhcp-leasefile=/etc/pihole/dhcp.leases #quiet-dhcp " > "${dhcpconfig}" + chmod a+r "${dhcpconfig}" if [[ "${PIHOLE_DOMAIN}" != "none" ]]; then echo "domain=${PIHOLE_DOMAIN}" >> "${dhcpconfig}" @@ -541,11 +542,13 @@ addAudit() do echo "${var}" >> /etc/pihole/auditlog.list done + chmod a+r /etc/pihole/auditlog.list } clearAudit() { echo -n "" > /etc/pihole/auditlog.list + chmod a+r /etc/pihole/auditlog.list } SetPrivacyLevel() { diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 94377647..2d85d854 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -194,7 +194,7 @@ if is_command apt-get ; then exit # exit the installer else printf " %b Enabling universe package repository for Ubuntu Bionic\\n" "${INFO}" - cp ${APT_SOURCES} ${APT_SOURCES}.backup # Backup current repo list + cp -p ${APT_SOURCES} ${APT_SOURCES}.backup # Backup current repo list printf " %b Backed up current configuration to %s\\n" "${TICK}" "${APT_SOURCES}.backup" add-apt-repository universe printf " %b Enabled %s\\n" "${TICK}" "'universe' repository" @@ -847,7 +847,7 @@ setIFCFG() { # Put the IP in variables without the CIDR notation printf -v CIDR "%s" "${IPV4_ADDRESS##*/}" # Backup existing interface configuration: - cp "${IFCFG_FILE}" "${IFCFG_FILE}".pihole.orig + cp -p "${IFCFG_FILE}" "${IFCFG_FILE}".pihole.orig # Build Interface configuration file using the GLOBAL variables we have { echo "# Configured via Pi-hole installer" @@ -1241,7 +1241,8 @@ version_check_dnsmasq() { printf "%b %b Backing up dnsmasq.conf to dnsmasq.conf.orig...\\n" "${OVER}" "${TICK}" printf " %b Restoring default dnsmasq.conf..." "${INFO}" # and replace it with the default - cp ${dnsmasq_original_config} ${dnsmasq_conf} + cp -p ${dnsmasq_original_config} ${dnsmasq_conf} + chmod a+r ${dnsmasq_conf} printf "%b %b Restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" # Otherwise, else @@ -1252,7 +1253,7 @@ version_check_dnsmasq() { # If a file cannot be found, printf " %b No dnsmasq.conf found... restoring default dnsmasq.conf..." "${INFO}" # restore the default one - cp ${dnsmasq_original_config} ${dnsmasq_conf} + cp -p ${dnsmasq_original_config} ${dnsmasq_conf} printf "%b %b No dnsmasq.conf found... restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" fi @@ -1260,9 +1261,11 @@ version_check_dnsmasq() { # Check to see if dnsmasq directory exists (it may not due to being a fresh install and dnsmasq no longer being a dependency) if [[ ! -d "/etc/dnsmasq.d" ]];then mkdir "/etc/dnsmasq.d" + chmod 755 "/etc/dnsmasq.d" fi # Copy the new Pi-hole DNS config file into the dnsmasq.d directory cp ${dnsmasq_pihole_01_snippet} ${dnsmasq_pihole_01_location} + chmod a+r ${dnsmasq_pihole_01_location} printf "%b %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf\\n" "${OVER}" "${TICK}" # Replace our placeholder values with the GLOBAL DNS variables that we populated earlier # First, swap in the interface to listen on @@ -1382,6 +1385,7 @@ installConfigs() { mkdir /etc/lighttpd # and set the owners chown "${USER}":root /etc/lighttpd + chmod 755 /etc/lighttpd # Otherwise, if the config file already exists elif [[ -f "/etc/lighttpd/lighttpd.conf" ]]; then # back up the original @@ -1389,8 +1393,10 @@ installConfigs() { fi # and copy in the config file Pi-hole needs cp ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} /etc/lighttpd/lighttpd.conf + chmod a+r /etc/lighttpd/lighttpd.conf # Make sure the external.conf file exists, as lighttpd v1.4.50 crashes without it touch /etc/lighttpd/external.conf + chmod a+r /etc/lighttpd/external.conf # if there is a custom block page in the html/pihole directory, replace 404 handler in lighttpd config if [[ -f "${PI_HOLE_BLOCKPAGE_DIR}/custom.php" ]]; then sed -i 's/^\(server\.error-handler-404\s*=\s*\).*$/\1"pihole\/custom\.php"/' /etc/lighttpd/lighttpd.conf @@ -1422,15 +1428,26 @@ install_manpage() { if [[ ! -d "/usr/local/share/man/man8" ]]; then # if not present, create man8 directory mkdir /usr/local/share/man/man8 + chown root:staff /usr/local/share/man/man8 + chmod a+r /usr/local/share/man/man8 + chmod a+x /usr/local/share/man/man8 + chmod g+s /usr/local/share/man/man8 fi if [[ ! -d "/usr/local/share/man/man5" ]]; then - # if not present, create man8 directory + # if not present, create man5 directory mkdir /usr/local/share/man/man5 + chown root:staff /usr/local/share/man/man5 + chmod a+r /usr/local/share/man/man5 + chmod a+x /usr/local/share/man/man5 + chmod g+s /usr/local/share/man/man5 fi # Testing complete, copy the files & update the man db cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 + chmod a+r /usr/local/share/man/man8/pihole.8 cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 + chmod a+r /usr/local/share/man/man8/pihole-FTL.8 cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.conf.5 /usr/local/share/man/man5/pihole-FTL.conf.5 + chmod a+r /usr/local/share/man/man5/pihole-FTL.conf.5 if mandb -q &>/dev/null; then # Updated successfully printf "%b %b man pages installed and database updated\\n" "${OVER}" "${TICK}" @@ -1711,6 +1728,9 @@ installCron() { printf "\\n %b %s..." "${INFO}" "${str}" # Copy the cron file over from the local repo cp ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.cron /etc/cron.d/pihole + # File must not be world or group writeable and must be owned by root + chmod 644 /etc/cron.d/pihole + chown root:root /etc/cron.d/pihole # Randomize gravity update time sed -i "s/59 1 /$((1 + RANDOM % 58)) $((3 + RANDOM % 2))/" /etc/cron.d/pihole # Randomize update checker time @@ -1818,6 +1838,7 @@ finalExports() { echo "INSTALL_WEB_INTERFACE=${INSTALL_WEB_INTERFACE}" echo "LIGHTTPD_ENABLED=${LIGHTTPD_ENABLED}" }>> "${setupVars}" + chmod 744 "${setupVars}" # Set the privacy level sed -i '/PRIVACYLEVEL/d' "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" @@ -1841,6 +1862,7 @@ installLogrotate() { printf "\\n %b %s..." "${INFO}" "${str}" # Copy the file over from the local repo cp ${PI_HOLE_LOCAL_REPO}/advanced/Templates/logrotate /etc/pihole/logrotate + chmod a+r /etc/pihole/logrotate # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. @@ -2431,6 +2453,7 @@ copy_to_install_log() { # Copy the contents of file descriptor 3 into the install log # Since we use color codes such as '\e[1;33m', they should be removed sed 's/\[[0-9;]\{1,5\}m//g' < /proc/$$/fd/3 > "${installLogLoc}" + chmod a+r "${installLogLoc}" } main() { @@ -2516,6 +2539,8 @@ main() { welcomeDialogs # Create directory for Pi-hole storage mkdir -p /etc/pihole/ + chmod a+r /ect/pihole/ + chmod a+x /etc/pihole/ # Determine available interfaces get_available_interfaces # Find interfaces and let the user choose one diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index d0a6dcf0..732fc246 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -153,7 +153,7 @@ removeNoPurge() { # Restore Resolved if [[ -e /etc/systemd/resolved.conf.orig ]]; then - ${SUDO} cp /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf + ${SUDO} cp -p /etc/systemd/resolved.conf.orig /etc/systemd/resolved.conf systemctl reload-or-restart systemd-resolved fi diff --git a/gravity.sh b/gravity.sh index 015764ac..6ae0e337 100755 --- a/gravity.sh +++ b/gravity.sh @@ -485,6 +485,7 @@ gravity_SortAndFilterConsolidatedList() { fi sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}" + chmod a+r "${piholeDir}/${preEventHorizon}" if [[ "${haveSourceUrls}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" @@ -509,6 +510,7 @@ gravity_Whitelist() { # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile comm -23 "${piholeDir}/${preEventHorizon}" <(sort "${whitelistFile}") > "${piholeDir}/${whitelistMatter}" + chmod a+r "${piholeDir}/${whitelistMatter}" echo -e "${OVER} ${INFO} ${str}" } @@ -561,6 +563,7 @@ gravity_ParseLocalDomains() { # Empty $localList if it already exists, otherwise, create it : > "${localList}" + chmod a+r "${localList}" gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}" @@ -581,8 +584,9 @@ gravity_ParseBlacklistDomains() { mv "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}" else # There was no whitelist file, so use preEventHorizon instead of whitelistMatter. - cp "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}" + cp -p "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}" fi + chmod a+r "${piholeDir}/${accretionDisc}" # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 ) @@ -592,6 +596,7 @@ gravity_ParseBlacklistDomains() { echo -e "\\n ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n ${output}" gravity_Cleanup "error" fi + chmod a+r "${adList}" } # Create user-added blacklist entries @@ -602,6 +607,7 @@ gravity_ParseUserDomains() { # Copy the file over as /etc/pihole/black.list so dnsmasq can use it cp "${blacklistFile}" "${blackList}" 2> /dev/null || \ echo -e "\\n ${CROSS} Unable to move ${blacklistFile##*/} to ${piholeDir}" + chmod a+r "${blackList}" } # Trap Ctrl-C diff --git a/pihole b/pihole index 84a5623f..59a99af1 100755 --- a/pihole +++ b/pihole @@ -151,10 +151,12 @@ Time: if [[ -e "${gravitylist}" ]]; then mv "${gravitylist}" "${gravitylist}.bck" echo "" > "${gravitylist}" + chmod a+r "${gravitylist}" fi if [[ -e "${blacklist}" ]]; then mv "${blacklist}" "${blacklist}.bck" echo "" > "${blacklist}" + chmod a+r "${blacklist}" fi if [[ $# > 1 ]]; then local error=false @@ -206,9 +208,11 @@ Time: if [[ -e "${gravitylist}.bck" ]]; then mv "${gravitylist}.bck" "${gravitylist}" + chmod a+r "${gravitylist}" fi if [[ -e "${blacklist}.bck" ]]; then mv "${blacklist}.bck" "${blacklist}" + chmod a+r "${blacklist}" fi sed -i "/BLOCKING_ENABLED=/d" "${setupVars}" echo "BLOCKING_ENABLED=true" >> "${setupVars}" From 8a92fb24c4d4e3caff4286ea8698becdfe986a58 Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Wed, 1 May 2019 11:20:26 +0200 Subject: [PATCH 058/207] Incorporates changes requested by @Mcat12 Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- advanced/Scripts/list.sh | 6 +++--- advanced/Scripts/piholeCheckout.sh | 6 +++--- advanced/Scripts/piholeLogFlush.sh | 4 ++-- advanced/Scripts/updatecheck.sh | 6 +++--- advanced/Scripts/webpage.sh | 6 +++--- automated install/basic-install.sh | 33 ++++++++++++------------------ gravity.sh | 12 +++++------ pihole | 8 ++++---- 8 files changed, 37 insertions(+), 44 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index e0455e0c..b4fffbaf 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -92,13 +92,13 @@ PoplistFile() { # Check whitelist file exists, and if not, create it if [[ ! -f "${whitelist}" ]]; then touch "${whitelist}" - chmod a+r "${whitelist}" + chmod 644 "${whitelist}" fi # Check blacklist file exists, and if not, create it if [[ ! -f "${blacklist}" ]]; then touch "${blacklist}" - chmod a+r "${blacklist}" + chmod 644 "${blacklist}" fi for dom in "${domList[@]}"; do @@ -244,7 +244,7 @@ NukeList() { cp -p "${listMain}" "${listMain}.bck~" # Empty out file echo "" > "${listMain}" - chmod a+r "${listMain}" + chmod 644 "${listMain}" fi } diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index a982289f..13fa3402 100644 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -90,7 +90,7 @@ checkout() { local path path="development/${binary}" echo "development" > /etc/pihole/ftlbranch - chmod a+r /etc/pihole/ftlbranch + chmod 644 /etc/pihole/ftlbranch elif [[ "${1}" == "master" ]] ; then # Shortcut to check out master branches echo -e " ${INFO} Shortcut \"master\" detected - checking out master branches..." @@ -105,7 +105,7 @@ checkout() { local path path="master/${binary}" echo "master" > /etc/pihole/ftlbranch - chmod a+r /etc/pihole/ftlbranch + chmod 644 /etc/pihole/ftlbranch elif [[ "${1}" == "core" ]] ; then str="Fetching branches from ${piholeGitUrl}" echo -ne " ${INFO} $str" @@ -168,7 +168,7 @@ checkout() { if check_download_exists "$path"; then echo " ${TICK} Branch ${2} exists" echo "${2}" > /etc/pihole/ftlbranch - chmod a+r /etc/pihole/ftlbranch + chmod 644 /etc/pihole/ftlbranch FTLinstall "${binary}" restart_service pihole-FTL enable_service pihole-FTL diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 1e4b7abe..51e94d7c 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -41,7 +41,7 @@ if [[ "$@" == *"once"* ]]; then # moved file (it will have the same file handler) cp -p /var/log/pihole.log /var/log/pihole.log.1 echo " " > /var/log/pihole.log - chmod a+r /var/log/pihole.log + chmod 644 /var/log/pihole.log fi else # Manual flushing @@ -54,7 +54,7 @@ else echo " " > /var/log/pihole.log if [ -f /var/log/pihole.log.1 ]; then echo " " > /var/log/pihole.log.1 - chmod a+r /var/log/pihole.log.1 + chmod 644 /var/log/pihole.log.1 fi fi # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 55fce328..972ab585 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -51,7 +51,7 @@ if [[ "$2" == "remote" ]]; then GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")" echo -n "${GITHUB_CORE_VERSION}" > "${GITHUB_VERSION_FILE}" - chmod a+r "${GITHUB_VERSION_FILE}" + chmod 644 "${GITHUB_VERSION_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")" @@ -67,7 +67,7 @@ else CORE_BRANCH="$(get_local_branch /etc/.pihole)" echo -n "${CORE_BRANCH}" > "${LOCAL_BRANCH_FILE}" - chmod a+r "${LOCAL_BRANCH_FILE}" + chmod 644 "${LOCAL_BRANCH_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then WEB_BRANCH="$(get_local_branch /var/www/html/admin)" @@ -81,7 +81,7 @@ else CORE_VERSION="$(get_local_version /etc/.pihole)" echo -n "${CORE_VERSION}" > "${LOCAL_VERSION_FILE}" - chmod a+r "${LOCAL_VERSION_FILE}" + chmod 644 "${LOCAL_VERSION_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then WEB_VERSION="$(get_local_version /var/www/html/admin)" diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index eba539a6..467eebd6 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -322,7 +322,7 @@ dhcp-option=option:router,${DHCP_ROUTER} dhcp-leasefile=/etc/pihole/dhcp.leases #quiet-dhcp " > "${dhcpconfig}" - chmod a+r "${dhcpconfig}" + chmod 644 "${dhcpconfig}" if [[ "${PIHOLE_DOMAIN}" != "none" ]]; then echo "domain=${PIHOLE_DOMAIN}" >> "${dhcpconfig}" @@ -542,13 +542,13 @@ addAudit() do echo "${var}" >> /etc/pihole/auditlog.list done - chmod a+r /etc/pihole/auditlog.list + chmod 644 /etc/pihole/auditlog.list } clearAudit() { echo -n "" > /etc/pihole/auditlog.list - chmod a+r /etc/pihole/auditlog.list + chmod 644 /etc/pihole/auditlog.list } SetPrivacyLevel() { diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 2d85d854..c1f69efb 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1242,7 +1242,7 @@ version_check_dnsmasq() { printf " %b Restoring default dnsmasq.conf..." "${INFO}" # and replace it with the default cp -p ${dnsmasq_original_config} ${dnsmasq_conf} - chmod a+r ${dnsmasq_conf} + chmod 644 ${dnsmasq_conf} printf "%b %b Restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" # Otherwise, else @@ -1265,7 +1265,7 @@ version_check_dnsmasq() { fi # Copy the new Pi-hole DNS config file into the dnsmasq.d directory cp ${dnsmasq_pihole_01_snippet} ${dnsmasq_pihole_01_location} - chmod a+r ${dnsmasq_pihole_01_location} + chmod 644 ${dnsmasq_pihole_01_location} printf "%b %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf\\n" "${OVER}" "${TICK}" # Replace our placeholder values with the GLOBAL DNS variables that we populated earlier # First, swap in the interface to listen on @@ -1393,10 +1393,10 @@ installConfigs() { fi # and copy in the config file Pi-hole needs cp ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} /etc/lighttpd/lighttpd.conf - chmod a+r /etc/lighttpd/lighttpd.conf + chmod 644 /etc/lighttpd/lighttpd.conf # Make sure the external.conf file exists, as lighttpd v1.4.50 crashes without it touch /etc/lighttpd/external.conf - chmod a+r /etc/lighttpd/external.conf + chmod 644 /etc/lighttpd/external.conf # if there is a custom block page in the html/pihole directory, replace 404 handler in lighttpd config if [[ -f "${PI_HOLE_BLOCKPAGE_DIR}/custom.php" ]]; then sed -i 's/^\(server\.error-handler-404\s*=\s*\).*$/\1"pihole\/custom\.php"/' /etc/lighttpd/lighttpd.conf @@ -1428,26 +1428,20 @@ install_manpage() { if [[ ! -d "/usr/local/share/man/man8" ]]; then # if not present, create man8 directory mkdir /usr/local/share/man/man8 - chown root:staff /usr/local/share/man/man8 - chmod a+r /usr/local/share/man/man8 - chmod a+x /usr/local/share/man/man8 - chmod g+s /usr/local/share/man/man8 + chmod 755 /usr/local/share/man/man8 fi if [[ ! -d "/usr/local/share/man/man5" ]]; then # if not present, create man5 directory mkdir /usr/local/share/man/man5 - chown root:staff /usr/local/share/man/man5 - chmod a+r /usr/local/share/man/man5 - chmod a+x /usr/local/share/man/man5 - chmod g+s /usr/local/share/man/man5 + chmod 755 /usr/local/share/man/man5 fi # Testing complete, copy the files & update the man db cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 - chmod a+r /usr/local/share/man/man8/pihole.8 + chmod 644 /usr/local/share/man/man8/pihole.8 cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 - chmod a+r /usr/local/share/man/man8/pihole-FTL.8 + chmod 644 /usr/local/share/man/man8/pihole-FTL.8 cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.conf.5 /usr/local/share/man/man5/pihole-FTL.conf.5 - chmod a+r /usr/local/share/man/man5/pihole-FTL.conf.5 + chmod 644 /usr/local/share/man/man5/pihole-FTL.conf.5 if mandb -q &>/dev/null; then # Updated successfully printf "%b %b man pages installed and database updated\\n" "${OVER}" "${TICK}" @@ -1838,7 +1832,7 @@ finalExports() { echo "INSTALL_WEB_INTERFACE=${INSTALL_WEB_INTERFACE}" echo "LIGHTTPD_ENABLED=${LIGHTTPD_ENABLED}" }>> "${setupVars}" - chmod 744 "${setupVars}" + chmod 644 "${setupVars}" # Set the privacy level sed -i '/PRIVACYLEVEL/d' "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" @@ -1862,7 +1856,7 @@ installLogrotate() { printf "\\n %b %s..." "${INFO}" "${str}" # Copy the file over from the local repo cp ${PI_HOLE_LOCAL_REPO}/advanced/Templates/logrotate /etc/pihole/logrotate - chmod a+r /etc/pihole/logrotate + chmod 644 /etc/pihole/logrotate # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. @@ -2453,7 +2447,7 @@ copy_to_install_log() { # Copy the contents of file descriptor 3 into the install log # Since we use color codes such as '\e[1;33m', they should be removed sed 's/\[[0-9;]\{1,5\}m//g' < /proc/$$/fd/3 > "${installLogLoc}" - chmod a+r "${installLogLoc}" + chmod 644 "${installLogLoc}" } main() { @@ -2539,8 +2533,7 @@ main() { welcomeDialogs # Create directory for Pi-hole storage mkdir -p /etc/pihole/ - chmod a+r /ect/pihole/ - chmod a+x /etc/pihole/ + chmod 755 /ect/pihole/ # Determine available interfaces get_available_interfaces # Find interfaces and let the user choose one diff --git a/gravity.sh b/gravity.sh index 6ae0e337..a2f2c782 100755 --- a/gravity.sh +++ b/gravity.sh @@ -485,7 +485,7 @@ gravity_SortAndFilterConsolidatedList() { fi sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}" - chmod a+r "${piholeDir}/${preEventHorizon}" + chmod 644 "${piholeDir}/${preEventHorizon}" if [[ "${haveSourceUrls}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" @@ -510,7 +510,7 @@ gravity_Whitelist() { # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile comm -23 "${piholeDir}/${preEventHorizon}" <(sort "${whitelistFile}") > "${piholeDir}/${whitelistMatter}" - chmod a+r "${piholeDir}/${whitelistMatter}" + chmod 644 "${piholeDir}/${whitelistMatter}" echo -e "${OVER} ${INFO} ${str}" } @@ -563,7 +563,7 @@ gravity_ParseLocalDomains() { # Empty $localList if it already exists, otherwise, create it : > "${localList}" - chmod a+r "${localList}" + chmod 644 "${localList}" gravity_ParseDomainsIntoHosts "${localList}.tmp" "${localList}" @@ -586,7 +586,7 @@ gravity_ParseBlacklistDomains() { # There was no whitelist file, so use preEventHorizon instead of whitelistMatter. cp -p "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}" fi - chmod a+r "${piholeDir}/${accretionDisc}" + chmod 644 "${piholeDir}/${accretionDisc}" # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 ) @@ -596,7 +596,7 @@ gravity_ParseBlacklistDomains() { echo -e "\\n ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n ${output}" gravity_Cleanup "error" fi - chmod a+r "${adList}" + chmod 644 "${adList}" } # Create user-added blacklist entries @@ -607,7 +607,7 @@ gravity_ParseUserDomains() { # Copy the file over as /etc/pihole/black.list so dnsmasq can use it cp "${blacklistFile}" "${blackList}" 2> /dev/null || \ echo -e "\\n ${CROSS} Unable to move ${blacklistFile##*/} to ${piholeDir}" - chmod a+r "${blackList}" + chmod 644 "${blackList}" } # Trap Ctrl-C diff --git a/pihole b/pihole index 59a99af1..541434a0 100755 --- a/pihole +++ b/pihole @@ -151,12 +151,12 @@ Time: if [[ -e "${gravitylist}" ]]; then mv "${gravitylist}" "${gravitylist}.bck" echo "" > "${gravitylist}" - chmod a+r "${gravitylist}" + chmod 644 "${gravitylist}" fi if [[ -e "${blacklist}" ]]; then mv "${blacklist}" "${blacklist}.bck" echo "" > "${blacklist}" - chmod a+r "${blacklist}" + chmod 644 "${blacklist}" fi if [[ $# > 1 ]]; then local error=false @@ -208,11 +208,11 @@ Time: if [[ -e "${gravitylist}.bck" ]]; then mv "${gravitylist}.bck" "${gravitylist}" - chmod a+r "${gravitylist}" + chmod 644 "${gravitylist}" fi if [[ -e "${blacklist}.bck" ]]; then mv "${blacklist}.bck" "${blacklist}" - chmod a+r "${blacklist}" + chmod 644 "${blacklist}" fi sed -i "/BLOCKING_ENABLED=/d" "${setupVars}" echo "BLOCKING_ENABLED=true" >> "${setupVars}" From 487718512dabdbdb4e3858eeed98c711f3de6fa7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 16:37:23 +0200 Subject: [PATCH 059/207] Remove DISTINCT attribute from views. They are not needed as the corresponding data is obtained from fields with UNIQUE attribute so duplications are already prevented at domain insertion. The elimination of the duplication check on imort through the views slightly reduces the load during import (I observed about 8 percent time saved on my NanoPi NEO). Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index dc0f23ed..dac0a50c 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -7,11 +7,11 @@ CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); INSERT INTO info VALUES("version","1"); -CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain +CREATE VIEW vw_gravity AS SELECT a.domain FROM gravity a WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); -CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain +CREATE VIEW vw_whitelist AS SELECT a.domain FROM whitelist a WHERE a.enabled == 1; CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist @@ -19,7 +19,7 @@ CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain +CREATE VIEW vw_blacklist AS SELECT a.domain FROM blacklist a WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist; CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist @@ -27,7 +27,7 @@ CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_regex AS SELECT DISTINCT a.domain +CREATE VIEW vw_regex AS SELECT a.domain FROM regex a WHERE a.enabled == 1; CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex @@ -35,7 +35,7 @@ CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_adlists AS SELECT DISTINCT a.address +CREATE VIEW vw_adlists AS SELECT a.address FROM adlists a WHERE a.enabled == 1; CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists From d5781fb1105d0c0ca1eda09c8e92f19ac135ff0c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 16:38:41 +0200 Subject: [PATCH 060/207] Fix extra space in front on END statement in CREATE TRIGGER commands Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index dac0a50c..0960018b 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -17,7 +17,7 @@ CREATE VIEW vw_whitelist AS SELECT a.domain CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist BEGIN UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; - END; + END; CREATE VIEW vw_blacklist AS SELECT a.domain FROM blacklist a @@ -25,7 +25,7 @@ CREATE VIEW vw_blacklist AS SELECT a.domain CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist BEGIN UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; - END; + END; CREATE VIEW vw_regex AS SELECT a.domain FROM regex a @@ -33,7 +33,7 @@ CREATE VIEW vw_regex AS SELECT a.domain CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex BEGIN UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; - END; + END; CREATE VIEW vw_adlists AS SELECT a.address FROM adlists a @@ -41,5 +41,5 @@ CREATE VIEW vw_adlists AS SELECT a.address CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists BEGIN UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; - END; + END; From bc9b62363895b8b6483e0f518ba46607b75f9909 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 16:56:16 +0200 Subject: [PATCH 061/207] Modify query.sh to use the gravity database Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) mode change 100644 => 100755 advanced/Scripts/query.sh diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh old mode 100644 new mode 100755 index 69a3c7a4..8eb7c404 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -11,7 +11,7 @@ # Globals piholeDir="/etc/pihole" -adListsList="$piholeDir/adlists.list" +gravityDBfile="${piholeDir}/gravity.db" wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf" options="$*" adlist="" @@ -73,11 +73,6 @@ Options: exit 0 fi -if [[ ! -e "$adListsList" ]]; then - echo -e "${COL_LIGHT_RED}The file $adListsList was not found${COL_NC}" - exit 1 -fi - # Handle valid options if [[ "${options}" == *"-bp"* ]]; then exact="exact"; blockpage=true @@ -185,12 +180,9 @@ if [[ -z "${exact}" ]]; then fi # Get adlist file content as array -if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then - for adlistUrl in $(< "${adListsList}"); do - if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then - adlists+=("${adlistUrl}") - fi - done +if [[ -n "${blockpage}" ]]; then + # Retrieve source URLs from gravity database + mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" fi # Print "Exact matches for" title From b372f808ddf1ff0fed03b1f314c0a85e487766cd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 17:06:14 +0200 Subject: [PATCH 062/207] Fix lint errors Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 2 +- gravity.sh | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 8eb7c404..a4ac895b 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -180,7 +180,7 @@ if [[ -z "${exact}" ]]; then fi # Get adlist file content as array -if [[ -n "${blockpage}" ]]; then +if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then # Retrieve source URLs from gravity database mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" fi diff --git a/gravity.sh b/gravity.sh index 331b632d..9f4065e7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -87,10 +87,11 @@ generate_gravity_database() { # Import domains from file and store them in the specified database table database_table_from_file() { # Define locals - local table="${1}" - local source="${2}" - local backup_path="${piholeDir}/migration_backup" - local backup_file="${backup_path}/$(basename "${2}")" + local table source backup_path backup_file + table="${1}" + source="${2}" + backup_path="${piholeDir}/migration_backup" + backup_file="${backup_path}/$(basename "${2}")" # Create database file if not present if [ ! -e "${gravityDBfile}" ]; then From 983117d788bd32ef1521a5c2da3088a2039e23bf Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 17:31:34 +0200 Subject: [PATCH 063/207] Add id (primary key, autoincrement) field to lists tables Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 28 ++++++++++++++++++++++++---- gravity.sh | 6 +++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 0960018b..65bd7448 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,7 +1,27 @@ -CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE regex (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE adlists (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); +CREATE TABLE whitelist (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT); +CREATE TABLE blacklist (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT); +CREATE TABLE regex (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT); +CREATE TABLE adlists (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + address TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT); CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); diff --git a/gravity.sh b/gravity.sh index 9f4065e7..c1eb0720 100755 --- a/gravity.sh +++ b/gravity.sh @@ -118,12 +118,16 @@ database_table_from_file() { inputfile="${source}" else # Apply format for white-, blacklist, regex, and adlists tables + local rowid + declare -i rowid + rowid=1 # Read file line by line grep -v '^ *#' < "${source}" | while IFS= read -r domain do # Only add non-empty lines if [[ ! -z "${domain}" ]]; then - echo "\"${domain}\",1,${timestamp},${timestamp}" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp}" >> "${tmpFile}" + rowid+=1 fi done inputfile="${tmpFile}" From 5e26e8245ba4f849577bb1ba232389ba00e59247 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 17:50:36 +0200 Subject: [PATCH 064/207] Order results of queries to list views by id key Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 65bd7448..d1651619 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -33,7 +33,9 @@ CREATE VIEW vw_gravity AS SELECT a.domain CREATE VIEW vw_whitelist AS SELECT a.domain FROM whitelist a - WHERE a.enabled == 1; + WHERE a.enabled == 1 + ORDER BY a.id; + CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist BEGIN UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; @@ -41,7 +43,9 @@ CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist CREATE VIEW vw_blacklist AS SELECT a.domain FROM blacklist a - WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist; + WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist + ORDER BY a.id; + CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist BEGIN UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; @@ -49,7 +53,9 @@ CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist CREATE VIEW vw_regex AS SELECT a.domain FROM regex a - WHERE a.enabled == 1; + WHERE a.enabled == 1 + ORDER BY a.id; + CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex BEGIN UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; @@ -57,7 +63,9 @@ CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex CREATE VIEW vw_adlists AS SELECT a.address FROM adlists a - WHERE a.enabled == 1; + WHERE a.enabled == 1 + ORDER BY a.id; + CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists BEGIN UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; From 540c57f755c0e6255c2416738478491494f692b4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 17:51:25 +0200 Subject: [PATCH 065/207] Add comment for automigrated list entries Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index c1eb0720..afd01aad 100755 --- a/gravity.sh +++ b/gravity.sh @@ -126,7 +126,7 @@ database_table_from_file() { do # Only add non-empty lines if [[ ! -z "${domain}" ]]; then - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp}" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" rowid+=1 fi done From 6c1e3a17f7534bb1ed7ea448401d11adc4b66406 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 17:54:29 +0200 Subject: [PATCH 066/207] Remove NOT NULL constraint on list ids as this is implied by PRIMARY KEY (even though most, if not all, examples on the web still use both attributes at the same time) Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index d1651619..4e78e026 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,22 +1,22 @@ -CREATE TABLE whitelist (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, +CREATE TABLE whitelist (id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE blacklist (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, +CREATE TABLE blacklist (id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE regex (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, +CREATE TABLE regex (id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT); -CREATE TABLE adlists (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, +CREATE TABLE adlists (id INTEGER PRIMARY KEY AUTOINCREMENT, address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), From 106f079afc24dd4f43699ca0a9f494f5e72dc850 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 17:57:23 +0200 Subject: [PATCH 067/207] Improve indentation (single tab) in SQLite template Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 71 ++++++++++++++++++++----------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 4e78e026..a8d48608 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,29 +1,48 @@ -CREATE TABLE whitelist (id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT); -CREATE TABLE blacklist (id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT); -CREATE TABLE regex (id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT); -CREATE TABLE adlists (id INTEGER PRIMARY KEY AUTOINCREMENT, - address TEXT UNIQUE NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 1, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT); -CREATE TABLE gravity (domain TEXT UNIQUE NOT NULL); -CREATE TABLE info (property TEXT NOT NULL, value TEXT NOT NULL); +CREATE TABLE whitelist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TABLE blacklist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TABLE regex +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TABLE adlists +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + address TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TABLE gravity +( + domain TEXT UNIQUE NOT NULL +); +CREATE TABLE info +( + property TEXT NOT NULL, + value TEXT NOT NULL +); INSERT INTO info VALUES("version","1"); From 2f698904e406fc978cb00bf4759aebbc22b14479 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 18:03:35 +0200 Subject: [PATCH 068/207] Use PRIMARY KEY instead of NOT NULL also in gravity and info tables Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index a8d48608..fd6d2b28 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -36,11 +36,11 @@ CREATE TABLE adlists ); CREATE TABLE gravity ( - domain TEXT UNIQUE NOT NULL + domain TEXT UNIQUE PRIMARY KEY ); CREATE TABLE info ( - property TEXT NOT NULL, + property TEXT PRIMARY KEY, value TEXT NOT NULL ); From 008e88b84be1f818fe6518d954ddc8b4a7aca63c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 18:04:31 +0200 Subject: [PATCH 069/207] Only migrate legacy list files when we create gravity.db in the same gravity run Signed-off-by: DL6ER --- gravity.sh | 51 ++++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 25 deletions(-) diff --git a/gravity.sh b/gravity.sh index afd01aad..02f41f86 100755 --- a/gravity.sh +++ b/gravity.sh @@ -93,12 +93,6 @@ database_table_from_file() { backup_path="${piholeDir}/migration_backup" backup_file="${backup_path}/$(basename "${2}")" - # Create database file if not present - if [ ! -e "${gravityDBfile}" ]; then - echo -e " ${INFO} Creating new gravity database" - generate_gravity_database - fi - # Truncate table output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 ) status="$?" @@ -155,25 +149,32 @@ database_table_from_file() { # Migrate pre-v5.0 list files to database-based Pi-hole versions migrate_to_database() { - if [[ -e "${adListFile}" ]]; then - # Store adlists domains in database - echo -e " ${INFO} Pi-hole upgrade: Moving content of ${adListFile} into database" - database_table_from_file "adlists" "${adListFile}" - fi - if [[ -e "${blacklistFile}" ]]; then - # Store blacklisted domains in database - echo -e " ${INFO} Pi-hole upgrade: Moving content of ${blacklistFile} into database" - database_table_from_file "blacklist" "${blacklistFile}" - fi - if [[ -e "${whitelistFile}" ]]; then - # Store whitelisted domains in database - echo -e " ${INFO} Pi-hole upgrade: Moving content of ${whitelistFile} into database" - database_table_from_file "whitelist" "${whitelistFile}" - fi - if [[ -e "${regexFile}" ]]; then - # Store regex domains in database - echo -e " ${INFO} Pi-hole upgrade: Moving content of ${regexFile} into database" - database_table_from_file "regex" "${regexFile}" + # Create database file if not present + if [ ! -e "${gravityDBfile}" ]; then + echo -e " ${INFO} Creating new gravity database" + generate_gravity_database + + # Migrate list files to new database + if [[ -e "${adListFile}" ]]; then + # Store adlists domains in database + echo -e " ${INFO} Migrating content of ${adListFile} into new database" + database_table_from_file "adlists" "${adListFile}" + fi + if [[ -e "${blacklistFile}" ]]; then + # Store blacklisted domains in database + echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" + database_table_from_file "blacklist" "${blacklistFile}" + fi + if [[ -e "${whitelistFile}" ]]; then + # Store whitelisted domains in database + echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" + database_table_from_file "whitelist" "${whitelistFile}" + fi + if [[ -e "${regexFile}" ]]; then + # Store regex domains in database + echo -e " ${INFO} Migrating content of ${regexFile} into new database" + database_table_from_file "regex" "${regexFile}" + fi fi } From 0cfecd31d3501c5a760207bd1780c855a90be2f9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 21:12:22 +0200 Subject: [PATCH 070/207] Invert logic to reduce nesting Signed-off-by: DL6ER --- gravity.sh | 52 +++++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/gravity.sh b/gravity.sh index 02f41f86..76d42f32 100755 --- a/gravity.sh +++ b/gravity.sh @@ -149,32 +149,34 @@ database_table_from_file() { # Migrate pre-v5.0 list files to database-based Pi-hole versions migrate_to_database() { - # Create database file if not present - if [ ! -e "${gravityDBfile}" ]; then - echo -e " ${INFO} Creating new gravity database" - generate_gravity_database + # Create database file only if not present + if [ -e "${gravityDBfile}" ]; then + return 0 + fi - # Migrate list files to new database - if [[ -e "${adListFile}" ]]; then - # Store adlists domains in database - echo -e " ${INFO} Migrating content of ${adListFile} into new database" - database_table_from_file "adlists" "${adListFile}" - fi - if [[ -e "${blacklistFile}" ]]; then - # Store blacklisted domains in database - echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" - database_table_from_file "blacklist" "${blacklistFile}" - fi - if [[ -e "${whitelistFile}" ]]; then - # Store whitelisted domains in database - echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" - database_table_from_file "whitelist" "${whitelistFile}" - fi - if [[ -e "${regexFile}" ]]; then - # Store regex domains in database - echo -e " ${INFO} Migrating content of ${regexFile} into new database" - database_table_from_file "regex" "${regexFile}" - fi + echo -e " ${INFO} Creating new gravity database" + generate_gravity_database + + # Migrate list files to new database + if [[ -e "${adListFile}" ]]; then + # Store adlists domains in database + echo -e " ${INFO} Migrating content of ${adListFile} into new database" + database_table_from_file "adlists" "${adListFile}" + fi + if [[ -e "${blacklistFile}" ]]; then + # Store blacklisted domains in database + echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" + database_table_from_file "blacklist" "${blacklistFile}" + fi + if [[ -e "${whitelistFile}" ]]; then + # Store whitelisted domains in database + echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" + database_table_from_file "whitelist" "${whitelistFile}" + fi + if [[ -e "${regexFile}" ]]; then + # Store regex domains in database + echo -e " ${INFO} Migrating content of ${regexFile} into new database" + database_table_from_file "regex" "${regexFile}" fi } From d46ff1d2d79dbd2cb6f2ee2f919e793a6371757b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 1 May 2019 22:08:30 +0200 Subject: [PATCH 071/207] Remove UNIQUE on gravity table as the only column is already PRIMARY KEY. Primary keys must contain UNIQUE values, and cannot contain NULL values Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index fd6d2b28..372a4a29 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -36,7 +36,7 @@ CREATE TABLE adlists ); CREATE TABLE gravity ( - domain TEXT UNIQUE PRIMARY KEY + domain TEXT PRIMARY KEY ); CREATE TABLE info ( From 0bc112ce52e717467835f052353c6c8283e8db20 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 2 May 2019 17:11:39 +0200 Subject: [PATCH 072/207] Query only those columns we actually use when showing domains on the CLI. Show date of last modification instead of addition date for domains. Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 73cb2878..fa81348b 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -170,7 +170,7 @@ Displaylist() { local list listname count num_pipes domain enabled status nicedate listname="${listType}" - data="$(sqlite3 "${gravityDBfile}" "SELECT * FROM ${listType};" 2> /dev/null)" + data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)" if [[ -z $data ]]; then echo -e "Not showing empty ${listname}" @@ -186,9 +186,9 @@ Displaylist() { num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")" # Extract domain and enabled status based on the obtained number of pipe characters - domain="$(cut -d'|' -f"-$((num_pipes-3))" <<< "${line}")" - enabled="$(cut -d'|' -f"$((num_pipes-2))" <<< "${line}")" - dateadded="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")" + domain="$(cut -d'|' -f"-$((num_pipes-1))" <<< "${line}")" + enabled="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")" + datemod="$(cut -d'|' -f"$((num_pipes+1))" <<< "${line}")" # Translate boolean status into human readable string if [[ "${enabled}" -eq 1 ]]; then @@ -198,9 +198,9 @@ Displaylist() { fi # Get nice representation of numerical date stored in database - nicedate=$(date --rfc-2822 -d "@${dateadded}") + nicedate=$(date --rfc-2822 -d "@${datemod}") - echo " ${count}: ${domain} (${status}, added ${nicedate})" + echo " ${count}: ${domain} (${status}, last modified ${nicedate})" count=$((count+1)) done <<< "${data}" fi From 2c3a27e9c7e03ac0464808ae0faf346568f135bc Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Fri, 3 May 2019 12:27:56 +0200 Subject: [PATCH 073/207] Replaces cp chmod with install and add read permission to repository files after checkout and pull Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 56 +++++++++++++----------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index c1f69efb..0d574f20 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -414,6 +414,8 @@ make_repo() { fi # Clone the repo and return the return code from this command git clone -q --depth 1 "${remoteRepo}" "${directory}" &> /dev/null || return $? + # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) + chmod -R a+r "${directory}" # Show a colored message showing it's status printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" # Always return 0? Not sure this is correct @@ -447,6 +449,8 @@ update_repo() { git pull --quiet &> /dev/null || return $? # Show a completion message printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" + # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) + chmod -R a+r "${directory}" # Move back into the original directory cd "${curdir}" &> /dev/null || return 1 return 0 @@ -494,6 +498,8 @@ resetRepo() { printf " %b %s..." "${INFO}" "${str}" # Use git to remove the local changes git reset --hard &> /dev/null || return $? + # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) + chmod -R a+r "${directory}" # And show the status printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" # Returning success anyway? @@ -1241,8 +1247,7 @@ version_check_dnsmasq() { printf "%b %b Backing up dnsmasq.conf to dnsmasq.conf.orig...\\n" "${OVER}" "${TICK}" printf " %b Restoring default dnsmasq.conf..." "${INFO}" # and replace it with the default - cp -p ${dnsmasq_original_config} ${dnsmasq_conf} - chmod 644 ${dnsmasq_conf} + install -D -m 644 -T ${dnsmasq_original_config} ${dnsmasq_conf} printf "%b %b Restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" # Otherwise, else @@ -1253,19 +1258,17 @@ version_check_dnsmasq() { # If a file cannot be found, printf " %b No dnsmasq.conf found... restoring default dnsmasq.conf..." "${INFO}" # restore the default one - cp -p ${dnsmasq_original_config} ${dnsmasq_conf} + install -D -m 644 -T ${dnsmasq_original_config} ${dnsmasq_conf} printf "%b %b No dnsmasq.conf found... restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" fi printf " %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf..." "${INFO}" # Check to see if dnsmasq directory exists (it may not due to being a fresh install and dnsmasq no longer being a dependency) if [[ ! -d "/etc/dnsmasq.d" ]];then - mkdir "/etc/dnsmasq.d" - chmod 755 "/etc/dnsmasq.d" + install -d -m 755 "/etc/dnsmasq.d" fi # Copy the new Pi-hole DNS config file into the dnsmasq.d directory - cp ${dnsmasq_pihole_01_snippet} ${dnsmasq_pihole_01_location} - chmod 644 ${dnsmasq_pihole_01_location} + install -D -m 644 -T ${dnsmasq_pihole_01_snippet} ${dnsmasq_pihole_01_location} printf "%b %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf\\n" "${OVER}" "${TICK}" # Replace our placeholder values with the GLOBAL DNS variables that we populated earlier # First, swap in the interface to listen on @@ -1381,19 +1384,15 @@ installConfigs() { if [[ "${INSTALL_WEB_SERVER}" == true ]]; then # and if the Web server conf directory does not exist, if [[ ! -d "/etc/lighttpd" ]]; then - # make it - mkdir /etc/lighttpd - # and set the owners - chown "${USER}":root /etc/lighttpd - chmod 755 /etc/lighttpd + # make it and set the owners + install -d -m 755 -o "${USER}" -g root /etc/lighttpd # Otherwise, if the config file already exists elif [[ -f "/etc/lighttpd/lighttpd.conf" ]]; then # back up the original mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig fi # and copy in the config file Pi-hole needs - cp ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} /etc/lighttpd/lighttpd.conf - chmod 644 /etc/lighttpd/lighttpd.conf + install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} /etc/lighttpd/lighttpd.conf # Make sure the external.conf file exists, as lighttpd v1.4.50 crashes without it touch /etc/lighttpd/external.conf chmod 644 /etc/lighttpd/external.conf @@ -1427,21 +1426,16 @@ install_manpage() { fi if [[ ! -d "/usr/local/share/man/man8" ]]; then # if not present, create man8 directory - mkdir /usr/local/share/man/man8 - chmod 755 /usr/local/share/man/man8 + install -d -m 755 /usr/local/share/man/man8 fi if [[ ! -d "/usr/local/share/man/man5" ]]; then # if not present, create man5 directory - mkdir /usr/local/share/man/man5 - chmod 755 /usr/local/share/man/man5 + install -d -m 755 /usr/local/share/man/man5 fi # Testing complete, copy the files & update the man db - cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 - chmod 644 /usr/local/share/man/man8/pihole.8 - cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 - chmod 644 /usr/local/share/man/man8/pihole-FTL.8 - cp ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.conf.5 /usr/local/share/man/man5/pihole-FTL.conf.5 - chmod 644 /usr/local/share/man/man5/pihole-FTL.conf.5 + install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole.8 /usr/local/share/man/man8/pihole.8 + install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.8 /usr/local/share/man/man8/pihole-FTL.8 + install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/manpages/pihole-FTL.conf.5 /usr/local/share/man/man5/pihole-FTL.conf.5 if mandb -q &>/dev/null; then # Updated successfully printf "%b %b man pages installed and database updated\\n" "${OVER}" "${TICK}" @@ -1671,7 +1665,7 @@ installPiholeWeb() { # Install the directory install -d -m 0755 ${PI_HOLE_BLOCKPAGE_DIR} # and the blockpage - install -D ${PI_HOLE_LOCAL_REPO}/advanced/{index,blockingpage}.* ${PI_HOLE_BLOCKPAGE_DIR}/ + install -D -m 644 ${PI_HOLE_LOCAL_REPO}/advanced/{index,blockingpage}.* ${PI_HOLE_BLOCKPAGE_DIR}/ # Remove superseded file if [[ -e "${PI_HOLE_BLOCKPAGE_DIR}/index.js" ]]; then @@ -1721,10 +1715,8 @@ installCron() { local str="Installing latest Cron script" printf "\\n %b %s..." "${INFO}" "${str}" # Copy the cron file over from the local repo - cp ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.cron /etc/cron.d/pihole # File must not be world or group writeable and must be owned by root - chmod 644 /etc/cron.d/pihole - chown root:root /etc/cron.d/pihole + install -D -m 644 -T -o root -g root ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.cron /etc/cron.d/pihole # Randomize gravity update time sed -i "s/59 1 /$((1 + RANDOM % 58)) $((3 + RANDOM % 2))/" /etc/cron.d/pihole # Randomize update checker time @@ -1855,8 +1847,7 @@ installLogrotate() { local str="Installing latest logrotate script" printf "\\n %b %s..." "${INFO}" "${str}" # Copy the file over from the local repo - cp ${PI_HOLE_LOCAL_REPO}/advanced/Templates/logrotate /etc/pihole/logrotate - chmod 644 /etc/pihole/logrotate + install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/Templates/logrotate /etc/pihole/logrotate # Different operating systems have different user / group # settings for logrotate that makes it impossible to create # a static logrotate file that will work with e.g. @@ -2120,6 +2111,8 @@ checkout_pull_branch() { printf " %b %s" "${INFO}" "$str" git checkout "${branch}" --quiet || return 1 printf "%b %b %s\\n" "${OVER}" "${TICK}" "$str" + # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) + chmod -R a+r "${directory}" git_pull=$(git pull || return 1) @@ -2532,8 +2525,7 @@ main() { # Display welcome dialogs welcomeDialogs # Create directory for Pi-hole storage - mkdir -p /etc/pihole/ - chmod 755 /ect/pihole/ + install -d -m 755 /etc/pihole/ # Determine available interfaces get_available_interfaces # Find interfaces and let the user choose one From 3aa838bbe47aee2f4ec804873fc2b267422d8f3c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 May 2019 12:47:25 +0200 Subject: [PATCH 074/207] Implement black- and whitelist searching with SQL statements. We use the ESCAPE clause in the LIKE query as the underscore "_" wildcard matches any single character but we want to suppress this behavior (underscores can be legitimate part of domains) Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 70 +++++++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index a4ac895b..9b7a0fab 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -102,29 +102,57 @@ if [[ -n "${str:-}" ]]; then exit 1 fi -# Scan Whitelist and Blacklist -lists="whitelist.txt blacklist.txt" -mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")" -if [[ -n "${results[*]}" ]]; then - wbMatch=true - # Loop through each result in order to print unique file title once - for result in "${results[@]}"; do - fileName="${result%%.*}" - if [[ -n "${blockpage}" ]]; then - echo "π ${result}" - exit 0 - elif [[ -n "${exact}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - else - # Only print filename title once per file - if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then +scanDatabaseTable() { + local domain table type querystr result + domain="${1}" + table="${2}" + type="${3:-}" + + # As underscores are legitimate parts of domains, we escape possible them when using the LIKE operator. + # Underscores are a SQLite wildcard matching exactly one character. We obviously want to suppress this + # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched + # as a literal underscore character. + case "${type}" in + "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; + * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\'";; + esac + + # Send prepared query to gravity database + result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null + if [[ -n "${result}" ]]; then + # Prepend listname (separated by a colon) if we found at least one result + # and output result + results="$(sed "s/^/${table}:/g;" <<< "${result}")" + else + # Output empty string as the database query didn't return any result + return + fi + mapfile -t results <<< "${results}" + if [[ -n "${results[*]}" ]]; then + wbMatch=true + # Loop through each result in order to print unique file title once + for result in "${results[@]}"; do + fileName="${result%%:*}" + if [[ -n "${blockpage}" ]]; then + echo "π ${result}" + exit 0 + elif [[ -n "${exact}" ]]; then echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - fileName_prev="${fileName}" + else + # Only print filename title once per file + if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" + fileName_prev="${fileName}" + fi + echo " ${result#*:}" fi - echo " ${result#*:}" - fi - done -fi + done + fi +} + +# Scan Whitelist and Blacklist +scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" +scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" # Scan Wildcards if [[ -e "${wildcardlist}" ]]; then From 6ba58896d23c5ac7bf497413192a1fdd225e03b1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 May 2019 13:15:30 +0200 Subject: [PATCH 075/207] Simplify code Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 48 +++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 9b7a0fab..66b55e87 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -103,7 +103,7 @@ if [[ -n "${str:-}" ]]; then fi scanDatabaseTable() { - local domain table type querystr result + local domain table type querystr result table_prev domain="${1}" table="${2}" type="${3:-}" @@ -119,35 +119,29 @@ scanDatabaseTable() { # Send prepared query to gravity database result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null - if [[ -n "${result}" ]]; then - # Prepend listname (separated by a colon) if we found at least one result - # and output result - results="$(sed "s/^/${table}:/g;" <<< "${result}")" - else - # Output empty string as the database query didn't return any result + if [[ -z "${result}" ]]; then + # Return early when we have no results return fi - mapfile -t results <<< "${results}" - if [[ -n "${results[*]}" ]]; then - wbMatch=true - # Loop through each result in order to print unique file title once - for result in "${results[@]}"; do - fileName="${result%%:*}" - if [[ -n "${blockpage}" ]]; then - echo "π ${result}" - exit 0 - elif [[ -n "${exact}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - else - # Only print filename title once per file - if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - fileName_prev="${fileName}" - fi - echo " ${result#*:}" + + wbMatch=true + mapfile -t results <<< "${result}" + # Loop through each result + for result in "${results[@]}"; do + if [[ -n "${blockpage}" ]]; then + echo "π ${result}" + exit 0 + elif [[ -n "${exact}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" + else + # Only print table name once + if [[ ! "${table}" == "${table_prev:-}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" + table_prev="${table}" fi - done - fi + echo " ${result}" + fi + done } # Scan Whitelist and Blacklist From f80fdd7e83b4bb23edb1671316f986b9ed791b68 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 May 2019 13:19:50 +0200 Subject: [PATCH 076/207] Improve comments Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 66b55e87..a7cc9dfb 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -108,10 +108,10 @@ scanDatabaseTable() { table="${2}" type="${3:-}" - # As underscores are legitimate parts of domains, we escape possible them when using the LIKE operator. - # Underscores are a SQLite wildcard matching exactly one character. We obviously want to suppress this + # As underscores are legitimate parts of domains, we escape them when using the LIKE operator. + # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched - # as a literal underscore character. + # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. case "${type}" in "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\'";; @@ -120,13 +120,14 @@ scanDatabaseTable() { # Send prepared query to gravity database result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null if [[ -z "${result}" ]]; then - # Return early when we have no results + # Return early when there are no matches in this table return fi + # Mark domain as having been white-/blacklist matched (global variable) wbMatch=true - mapfile -t results <<< "${result}" # Loop through each result + mapfile -t results <<< "${result}" for result in "${results[@]}"; do if [[ -n "${blockpage}" ]]; then echo "π ${result}" From 5246b3e49672edb7e397418160ca2c327cef0ccc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 May 2019 13:24:36 +0200 Subject: [PATCH 077/207] Explicitly escape backslash in ESCAPE clause. This has been suggested by Stickler bot. Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index a7cc9dfb..d4de380c 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -114,7 +114,7 @@ scanDatabaseTable() { # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. case "${type}" in "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; - * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\'";; + * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; esac # Send prepared query to gravity database From a904c183dfc2de7a503466f5ff39d1af3a4d1aca Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 4 May 2019 18:25:11 +0200 Subject: [PATCH 078/207] Use printf to escape domain content. This prevents possible SQL injection issues Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index d4de380c..d2e16e79 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -104,7 +104,7 @@ fi scanDatabaseTable() { local domain table type querystr result table_prev - domain="${1}" + domain="$(printf "%q" "${1}")" table="${2}" type="${3:-}" From 69c06ba6fe18e07f3b5611e8d849a996ff1931f1 Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Sun, 5 May 2019 00:32:10 +0200 Subject: [PATCH 079/207] Sets permissions for dnsmaq.conf Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 0d574f20..208bfe7c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1366,6 +1366,7 @@ installConfigs() { # Format: Name;Primary IPv4;Secondary IPv4;Primary IPv6;Secondary IPv6 # Some values may be empty (for example: DNS servers without IPv6 support) echo "${DNS_SERVERS}" > "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" + chmod 644 "${PI_HOLE_CONFIG_DIR}/dns-servers.conf" # Install empty file if it does not exist if [[ ! -r "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" ]]; then @@ -1692,7 +1693,7 @@ installPiholeWeb() { local str="Installing sudoer file" printf "\\n %b %s..." "${INFO}" "${str}" # Make the .d directory if it doesn't exist - mkdir -p /etc/sudoers.d/ + install -d -m 755 /etc/sudoers.d/ # and copy in the pihole sudoers file install -m 0640 ${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole.sudo /etc/sudoers.d/pihole # Add lighttpd user (OS dependent) to sudoers file @@ -2259,6 +2260,7 @@ disable_dnsmasq() { fi # Create /etc/dnsmasq.conf echo "conf-dir=/etc/dnsmasq.d" > "${conffile}" + chmod 644 "${conffile}" } get_binary_name() { From a25f331e83c94320b634b7954d071a0688ff4730 Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Sun, 5 May 2019 09:20:49 +0200 Subject: [PATCH 080/207] Adds chmod 644 for /var/www and /var/www/html Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 208bfe7c..79a6c94a 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -867,6 +867,8 @@ setIFCFG() { echo "DNS2=$PIHOLE_DNS_2" echo "USERCTL=no" }> "${IFCFG_FILE}" + chmod 644 "${IFCFG_FILE}" + chown root:root "${IFCFG_FILE}" # Use ip to immediately set the new address ip addr replace dev "${PIHOLE_INTERFACE}" "${IPV4_ADDRESS}" # If NetworkMangler command line interface exists and ready to mangle, @@ -1896,6 +1898,8 @@ installPihole() { # make the Web directory if necessary install -d -m 0755 ${webroot} fi + chmod a+rx /var/www + chmod a+rx /var/www/html if [[ "${INSTALL_WEB_SERVER}" == true ]]; then # Set the owner and permissions From e19adccd9c623eeba20e46a6347108e917af133e Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Sun, 5 May 2019 22:34:13 +0200 Subject: [PATCH 081/207] Moves chmod for /var/www/html into INSTALL_WEB_SERVER block Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 79a6c94a..da1a6f9b 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1898,13 +1898,14 @@ installPihole() { # make the Web directory if necessary install -d -m 0755 ${webroot} fi - chmod a+rx /var/www - chmod a+rx /var/www/html if [[ "${INSTALL_WEB_SERVER}" == true ]]; then # Set the owner and permissions chown ${LIGHTTPD_USER}:${LIGHTTPD_GROUP} ${webroot} chmod 0775 ${webroot} + # Repair permissions if /var/www/html is not world readable + chmod a+rx /var/www + chmod a+rx /var/www/html # Give pihole access to the Web server group usermod -a -G ${LIGHTTPD_GROUP} pihole # If the lighttpd command is executable, From 4d23b3267f1ff16a4392c481f7e84473dd42649c Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Tue, 7 May 2019 10:46:16 +0200 Subject: [PATCH 082/207] Adds X to chmod for git operations Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index da1a6f9b..585896c1 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -415,7 +415,7 @@ make_repo() { # Clone the repo and return the return code from this command git clone -q --depth 1 "${remoteRepo}" "${directory}" &> /dev/null || return $? # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) - chmod -R a+r "${directory}" + chmod -R a+rX "${directory}" # Show a colored message showing it's status printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" # Always return 0? Not sure this is correct @@ -450,7 +450,7 @@ update_repo() { # Show a completion message printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) - chmod -R a+r "${directory}" + chmod -R a+rX "${directory}" # Move back into the original directory cd "${curdir}" &> /dev/null || return 1 return 0 @@ -499,7 +499,7 @@ resetRepo() { # Use git to remove the local changes git reset --hard &> /dev/null || return $? # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) - chmod -R a+r "${directory}" + chmod -R a+rX "${directory}" # And show the status printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" # Returning success anyway? @@ -2118,7 +2118,7 @@ checkout_pull_branch() { git checkout "${branch}" --quiet || return 1 printf "%b %b %s\\n" "${OVER}" "${TICK}" "$str" # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) - chmod -R a+r "${directory}" + chmod -R a+rX "${directory}" git_pull=$(git pull || return 1) From 2e6198077d8f505415b8f494357eb40cdec12277 Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Thu, 9 May 2019 12:23:44 +0200 Subject: [PATCH 083/207] chmod 644 adlists.list Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 585896c1..00c5c95a 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1193,6 +1193,7 @@ chooseBlocklists() { do appendToListsFile "${choice}" done + chmod 644 "${adlistFile}" } # Accept a string parameter, it must be one of the default lists From a3d2a1062fe12f4da89c21274b8228c133dd5bfa Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Fri, 10 May 2019 11:13:19 +0200 Subject: [PATCH 084/207] Makes blocklists world readable Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- gravity.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/gravity.sh b/gravity.sh index a2f2c782..445274ce 100755 --- a/gravity.sh +++ b/gravity.sh @@ -328,6 +328,7 @@ gravity_DownloadBlocklistFromUrl() { # Parse source files into domains format gravity_ParseFileIntoDomains() { local source="${1}" destination="${2}" firstLine abpFilter + chmod 644 "${source}" # Determine if we are parsing a consolidated list if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then @@ -345,6 +346,7 @@ gravity_ParseFileIntoDomains() { sed -r '/(\/|#).*$/d' | \ sed -r 's/^.*\s+//g' | \ sed -r '/([^\.]+\.)+[^\.]{2,}/!d' > "${destination}" + chmod 644 "${destination}" return 0 fi @@ -375,6 +377,7 @@ gravity_ParseFileIntoDomains() { if($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } if($0) { print $0 } }' "${source}" > "${destination}" + chmod 644 "${destination}" # Determine if there are Adblock exception rules # https://adblockplus.org/filters @@ -391,6 +394,7 @@ gravity_ParseFileIntoDomains() { # Remove exceptions comm -23 "${destination}" <(sort "${destination}.exceptionsFile.tmp") > "${source}" + chmod 644 "${source}" mv "${source}" "${destination}" fi @@ -427,6 +431,7 @@ gravity_ParseFileIntoDomains() { gravity_Cleanup "error" fi fi + chmod 644 "${destination}" } # Create (unfiltered) "Matter and Light" consolidated list @@ -440,6 +445,7 @@ gravity_ConsolidateDownloadedBlocklists() { # Empty $matterAndLight if it already exists, otherwise, create it : > "${piholeDir}/${matterAndLight}" + chmod 644 "${piholeDir}/${matterAndLight}" # Loop through each *.domains file for i in "${activeDomains[@]}"; do From e076db5a4e1e15cb9d8e8707bf2a5da7be125489 Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Fri, 10 May 2019 11:19:28 +0200 Subject: [PATCH 085/207] Changes ownership of macvendor.db to pihole Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 00c5c95a..a00aee16 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2216,6 +2216,8 @@ FTLinstall() { # Before stopping FTL, we download the macvendor database curl -sSL "https://ftl.pi-hole.net/macvendor.db" -o "${PI_HOLE_CONFIG_DIR}/macvendor.db" || true + chmod 644 "${PI_HOLE_CONFIG_DIR}/macvendor.db" + chown pihole:pihole "${PI_HOLE_CONFIG_DIR}/macvendor.db" # Stop pihole-FTL service if available stop_service pihole-FTL &> /dev/null From 225285cb977ced4047de542978749c35a3ee43bf Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Sat, 11 May 2019 11:14:07 +0200 Subject: [PATCH 086/207] Removes chmod for ${source} and superfluous chmod for ${destination} Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- gravity.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/gravity.sh b/gravity.sh index 445274ce..8fdfa5d7 100755 --- a/gravity.sh +++ b/gravity.sh @@ -328,7 +328,6 @@ gravity_DownloadBlocklistFromUrl() { # Parse source files into domains format gravity_ParseFileIntoDomains() { local source="${1}" destination="${2}" firstLine abpFilter - chmod 644 "${source}" # Determine if we are parsing a consolidated list if [[ "${source}" == "${piholeDir}/${matterAndLight}" ]]; then @@ -377,7 +376,6 @@ gravity_ParseFileIntoDomains() { if($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } if($0) { print $0 } }' "${source}" > "${destination}" - chmod 644 "${destination}" # Determine if there are Adblock exception rules # https://adblockplus.org/filters @@ -394,7 +392,6 @@ gravity_ParseFileIntoDomains() { # Remove exceptions comm -23 "${destination}" <(sort "${destination}.exceptionsFile.tmp") > "${source}" - chmod 644 "${source}" mv "${source}" "${destination}" fi From 3f90261520aae689e213637bc27d0754ab0ce97b Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Sun, 12 May 2019 11:25:48 +0200 Subject: [PATCH 087/207] Moves chmod after every creation of ${destination} instead of end of function Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- gravity.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 8fdfa5d7..2a22b792 100755 --- a/gravity.sh +++ b/gravity.sh @@ -376,6 +376,7 @@ gravity_ParseFileIntoDomains() { if($0 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/) { $0="" } if($0) { print $0 } }' "${source}" > "${destination}" + chmod 644 "${destination}" # Determine if there are Adblock exception rules # https://adblockplus.org/filters @@ -393,6 +394,7 @@ gravity_ParseFileIntoDomains() { # Remove exceptions comm -23 "${destination}" <(sort "${destination}.exceptionsFile.tmp") > "${source}" mv "${source}" "${destination}" + chmod 644 "${destination}" fi echo -e "${OVER} ${TICK} Format: Adblock" @@ -416,11 +418,13 @@ gravity_ParseFileIntoDomains() { # Print if nonempty length { print } ' "${source}" 2> /dev/null > "${destination}" + chmod 644 "${destination}" echo -e "${OVER} ${TICK} Format: URL" else # Default: Keep hosts/domains file in same format as it was downloaded output=$( { mv "${source}" "${destination}"; } 2>&1 ) + chmod 644 "${destination}" if [[ ! -e "${destination}" ]]; then echo -e "\\n ${CROSS} Unable to move tmp file to ${piholeDir} @@ -428,7 +432,6 @@ gravity_ParseFileIntoDomains() { gravity_Cleanup "error" fi fi - chmod 644 "${destination}" } # Create (unfiltered) "Matter and Light" consolidated list From efbd42f7b5e7a9532aeefa13ff1965cb62d67981 Mon Sep 17 00:00:00 2001 From: pvogt09 <50047961+pvogt09@users.noreply.github.com> Date: Sun, 12 May 2019 11:27:25 +0200 Subject: [PATCH 088/207] Moves creation of pihole user before FTL installation Signed-off-by: pvogt09 <50047961+pvogt09@users.noreply.github.com> --- automated install/basic-install.sh | 5 ++--- test/test_automated_install.py | 9 +++++++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a00aee16..9520c44c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1890,9 +1890,6 @@ accountForRefactor() { # Install base files and web interface installPihole() { - # Create the pihole user - create_pihole_user - # If the user wants to install the Web interface, if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then if [[ ! -d "${webroot}" ]]; then @@ -2591,6 +2588,8 @@ main() { else LIGHTTPD_ENABLED=false fi + # Create the pihole user + create_pihole_user # Check if FTL is installed - do this early on as FTL is a hard dependency for Pi-hole if ! FTLdetect; then printf " %b FTL Engine not installed\\n" "${CROSS}" diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 853048d1..282c627d 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -398,6 +398,7 @@ def test_FTL_detect_aarch64_no_errors(Pihole): ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLdetect ''') expected_stdout = info_box + ' FTL Checks...' @@ -418,6 +419,7 @@ def test_FTL_detect_armv6l_no_errors(Pihole): mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLdetect ''') expected_stdout = info_box + ' FTL Checks...' @@ -439,6 +441,7 @@ def test_FTL_detect_armv7l_no_errors(Pihole): mock_command('ldd', {'/bin/ls': ('/lib/ld-linux-armhf.so.3', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLdetect ''') expected_stdout = info_box + ' FTL Checks...' @@ -455,6 +458,7 @@ def test_FTL_detect_x86_64_no_errors(Pihole): ''' detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLdetect ''') expected_stdout = info_box + ' FTL Checks...' @@ -471,6 +475,7 @@ def test_FTL_detect_unknown_no_errors(Pihole): mock_command('uname', {'-m': ('mips', '0')}, Pihole) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLdetect ''') expected_stdout = 'Not able to detect architecture (unknown: mips)' @@ -484,6 +489,7 @@ def test_FTL_download_aarch64_no_errors(Pihole): download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh binary="pihole-FTL-aarch64-linux-gnu" + create_pihole_user FTLinstall ''') expected_stdout = tick_box + ' Downloading and Installing FTL' @@ -498,6 +504,7 @@ def test_FTL_download_unknown_fails_no_errors(Pihole): download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh binary="pihole-FTL-mips" + create_pihole_user FTLinstall ''') expected_stdout = cross_box + ' Downloading and Installing FTL' @@ -514,6 +521,7 @@ def test_FTL_download_binary_unset_no_errors(Pihole): ''' download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLinstall ''') expected_stdout = cross_box + ' Downloading and Installing FTL' @@ -530,6 +538,7 @@ def test_FTL_binary_installed_and_responsive_no_errors(Pihole): ''' installed_binary = Pihole.run(''' source /opt/pihole/basic-install.sh + create_pihole_user FTLdetect pihole-FTL version ''') From ab2f8a0c09ecdb1ce1499f48267c05021e2bb29b Mon Sep 17 00:00:00 2001 From: David Haguenauer Date: Fri, 10 May 2019 10:13:23 -0400 Subject: [PATCH 089/207] Quote variables in basic-install.sh This greatly reduces the number of warnings emitted by ShellCheck, and in turn should make it more likely that errors are caught in the future. Signed-off-by: David Haguenauer --- automated install/basic-install.sh | 116 ++++++++++++++--------------- 1 file changed, 58 insertions(+), 58 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 18c47485..35d3305c 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -118,7 +118,7 @@ done # If the color table file exists, if [[ -f "${coltable}" ]]; then # source it - source ${coltable} + source "${coltable}" # Otherwise, else # Set these values so the installer can still run in color @@ -182,14 +182,14 @@ if is_command apt-get ; then # A variable to store the command used to update the package cache UPDATE_PKG_CACHE="${PKG_MANAGER} update" # An array for something... - PKG_INSTALL=(${PKG_MANAGER} --yes --no-install-recommends install) + PKG_INSTALL=("${PKG_MANAGER}" --yes --no-install-recommends install) # grep -c will return 1 retVal on 0 matches, block this throwing the set -e with an OR TRUE PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" # Some distros vary slightly so these fixes for dependencies may apply # on Ubuntu 18.04.1 LTS we need to add the universe repository to gain access to dialog and dhcpcd5 APT_SOURCES="/etc/apt/sources.list" if awk 'BEGIN{a=1;b=0}/bionic main/{a=0}/bionic.*universe/{b=1}END{exit a + b}' ${APT_SOURCES}; then - if ! whiptail --defaultno --title "Dependencies Require Update to Allowed Repositories" --yesno "Would you like to enable 'universe' repository?\\n\\nThis repository is required by the following packages:\\n\\n- dhcpcd5\\n- dialog" ${r} ${c}; then + if ! whiptail --defaultno --title "Dependencies Require Update to Allowed Repositories" --yesno "Would you like to enable 'universe' repository?\\n\\nThis repository is required by the following packages:\\n\\n- dhcpcd5\\n- dialog" "${r}" "${c}"; then printf " %b Aborting installation: dependencies could not be installed.\\n" "${CROSS}" exit # exit the installer else @@ -201,7 +201,7 @@ if is_command apt-get ; then fi fi # Debian 7 doesn't have iproute2 so if the dry run install is successful, - if ${PKG_MANAGER} install --dry-run iproute2 > /dev/null 2>&1; then + if "${PKG_MANAGER}" install --dry-run iproute2 > /dev/null 2>&1; then # we can install it iproute_pkg="iproute2" # Otherwise, @@ -222,7 +222,7 @@ if is_command apt-get ; then # Check if installed php is v 7.0, or newer to determine packages to install if [[ "$phpInsNewer" != true ]]; then # Prefer the php metapackage if it's there - if ${PKG_MANAGER} install --dry-run php > /dev/null 2>&1; then + if "${PKG_MANAGER}" install --dry-run php > /dev/null 2>&1; then phpVer="php" # fall back on the php5 packages else @@ -233,19 +233,19 @@ if is_command apt-get ; then phpVer="php$phpInsMajor.$phpInsMinor" fi # We also need the correct version for `php-sqlite` (which differs across distros) - if ${PKG_MANAGER} install --dry-run ${phpVer}-sqlite3 > /dev/null 2>&1; then + if "${PKG_MANAGER}" install --dry-run "${phpVer}"-sqlite3 > /dev/null 2>&1; then phpSqlite="sqlite3" else phpSqlite="sqlite" fi # Since our install script is so large, we need several other programs to successfully get a machine provisioned # These programs are stored in an array so they can be looped through later - INSTALLER_DEPS=(apt-utils dialog debconf dhcpcd5 git ${iproute_pkg} whiptail) + INSTALLER_DEPS=(apt-utils dialog debconf dhcpcd5 git "${iproute_pkg}" whiptail) # Pi-hole itself has several dependencies that also need to be installed PIHOLE_DEPS=(cron curl dnsutils iputils-ping lsof netcat psmisc sudo unzip wget idn2 sqlite3 libcap2-bin dns-root-data resolvconf libcap2) # The Web dashboard has some that also need to be installed # It's useful to separate the two since our repos are also setup as "Core" code and "Web" code - PIHOLE_WEB_DEPS=(lighttpd ${phpVer}-common ${phpVer}-cgi ${phpVer}-${phpSqlite}) + PIHOLE_WEB_DEPS=(lighttpd "${phpVer}"-common "${phpVer}"-cgi "${phpVer}-${phpSqlite}") # The Web server user, LIGHTTPD_USER="www-data" # group, @@ -281,7 +281,7 @@ elif is_command rpm ; then # Fedora and family update cache on every PKG_INSTALL call, no need for a separate update. UPDATE_PKG_CACHE=":" - PKG_INSTALL=(${PKG_MANAGER} install -y) + PKG_INSTALL=("${PKG_MANAGER}" install -y) PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" INSTALLER_DEPS=(dialog git iproute newt procps-ng which) PIHOLE_DEPS=(bind-utils cronie curl findutils nmap-ncat sudo unzip wget libidn2 psmisc sqlite libcap) @@ -319,7 +319,7 @@ elif is_command rpm ; then # The default php on CentOS 7.x is 5.4 which is EOL # Check if the version of PHP available via installed repositories is >= to PHP 7 - AVAILABLE_PHP_VERSION=$(${PKG_MANAGER} info php | grep -i version | grep -o '[0-9]\+' | head -1) + AVAILABLE_PHP_VERSION=$("${PKG_MANAGER}" info php | grep -i version | grep -o '[0-9]\+' | head -1) if [[ $AVAILABLE_PHP_VERSION -ge $SUPPORTED_CENTOS_PHP_VERSION ]]; then # Since PHP 7 is available by default, install via default PHP package names : # do nothing as PHP is current @@ -329,7 +329,7 @@ elif is_command rpm ; then rpm -q ${REMI_PKG} &> /dev/null || rc=$? if [[ $rc -ne 0 ]]; then # The PHP version available via default repositories is older than version 7 - if ! whiptail --defaultno --title "PHP 7 Update (recommended)" --yesno "PHP 7.x is recommended for both security and language features.\\nWould you like to install PHP7 via Remi's RPM repository?\\n\\nSee: https://rpms.remirepo.net for more information" ${r} ${c}; then + if ! whiptail --defaultno --title "PHP 7 Update (recommended)" --yesno "PHP 7.x is recommended for both security and language features.\\nWould you like to install PHP7 via Remi's RPM repository?\\n\\nSee: https://rpms.remirepo.net for more information" "${r}" "${c}"; then # User decided to NOT update PHP from REMI, attempt to install the default available PHP version printf " %b User opt-out of PHP 7 upgrade on CentOS. Deprecated PHP may be in use.\\n" "${INFO}" : # continue with unsupported php version @@ -352,7 +352,7 @@ elif is_command rpm ; then fi else # Warn user of unsupported version of Fedora or CentOS - if ! whiptail --defaultno --title "Unsupported RPM based distribution" --yesno "Would you like to continue installation on an unsupported RPM based distribution?\\n\\nPlease ensure the following packages have been installed manually:\\n\\n- lighttpd\\n- lighttpd-fastcgi\\n- PHP version 7+" ${r} ${c}; then + if ! whiptail --defaultno --title "Unsupported RPM based distribution" --yesno "Would you like to continue installation on an unsupported RPM based distribution?\\n\\nPlease ensure the following packages have been installed manually:\\n\\n- lighttpd\\n- lighttpd-fastcgi\\n- PHP version 7+" "${r}" "${c}"; then printf " %b Aborting installation due to unsupported RPM based distribution\\n" "${CROSS}" exit # exit the installer else @@ -543,15 +543,15 @@ get_available_interfaces() { # A function for displaying the dialogs the user sees when first running the installer welcomeDialogs() { # Display the welcome dialog using an appropriately sized window via the calculation conducted earlier in the script - whiptail --msgbox --backtitle "Welcome" --title "Pi-hole automated installer" "\\n\\nThis installer will transform your device into a network-wide ad blocker!" ${r} ${c} + whiptail --msgbox --backtitle "Welcome" --title "Pi-hole automated installer" "\\n\\nThis installer will transform your device into a network-wide ad blocker!" "${r}" "${c}" # Request that users donate if they enjoy the software since we all work on it in our free time - whiptail --msgbox --backtitle "Plea" --title "Free and open source" "\\n\\nThe Pi-hole is free, but powered by your donations: http://pi-hole.net/donate" ${r} ${c} + whiptail --msgbox --backtitle "Plea" --title "Free and open source" "\\n\\nThe Pi-hole is free, but powered by your donations: http://pi-hole.net/donate" "${r}" "${c}" # Explain the need for a static address whiptail --msgbox --backtitle "Initiating network interface" --title "Static IP Needed" "\\n\\nThe Pi-hole is a SERVER so it needs a STATIC IP ADDRESS to function properly. -In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." ${r} ${c} +In the next section, you can choose to use your current network settings (DHCP) or to manually edit them." "${r}" "${c}" } # We need to make sure there is enough space before installing, so there is a function to check this @@ -638,7 +638,7 @@ chooseInterface() { # Feed the available interfaces into this while loop done <<< "${availableInterfaces}" # The whiptail command that will be run, stored in a variable - chooseInterfaceCmd=(whiptail --separate-output --radiolist "Choose An Interface (press space to select)" ${r} ${c} ${interfaceCount}) + chooseInterfaceCmd=(whiptail --separate-output --radiolist "Choose An Interface (press space to select)" "${r}" "${c}" "${interfaceCount}") # Now run the command using the interfaces saved into the array chooseInterfaceOptions=$("${chooseInterfaceCmd[@]}" "${interfacesArray[@]}" 2>&1 >/dev/tty) || \ # If the user chooses Cancel, exit @@ -719,7 +719,7 @@ useIPv6dialog() { # If the IPV6_ADDRESS contains a value if [[ ! -z "${IPV6_ADDRESS}" ]]; then # Display that IPv6 is supported and will be used - whiptail --msgbox --backtitle "IPv6..." --title "IPv6 Supported" "$IPV6_ADDRESS will be used to block ads." ${r} ${c} + whiptail --msgbox --backtitle "IPv6..." --title "IPv6 Supported" "$IPV6_ADDRESS will be used to block ads." "${r}" "${c}" fi } @@ -729,7 +729,7 @@ use4andor6() { local useIPv4 local useIPv6 # Let use select IPv4 and/or IPv6 via a checklist - cmd=(whiptail --separate-output --checklist "Select Protocols (press space to select)" ${r} ${c} 2) + cmd=(whiptail --separate-output --checklist "Select Protocols (press space to select)" "${r}" "${c}" 2) # In an array, show the options available: # IPv4 (on by default) options=(IPv4 "Block ads over IPv4" on @@ -778,11 +778,11 @@ getStaticIPv4Settings() { # This is useful for users that are using DHCP reservations; then we can just use the information gathered via our functions if whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Do you want to use your current network settings as a static address? IP address: ${IPV4_ADDRESS} - Gateway: ${IPv4gw}" ${r} ${c}; then + Gateway: ${IPv4gw}" "${r}" "${c}"; then # If they choose yes, let the user know that the IP address will not be available via DHCP and may cause a conflict. whiptail --msgbox --backtitle "IP information" --title "FYI: IP Conflict" "It is possible your router could still try to assign this IP to a device, which would cause a conflict. But in most cases the router is smart enough to not do that. If you are worried, either manually set the address, or modify the DHCP reservation pool so it does not include the IP you want. -It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." ${r} ${c} +It is also possible to use a DHCP reservation, but if you are going to do that, you might as well set a static address." "${r}" "${c}" # Nothing else to do since the variables are already set above else # Otherwise, we need to ask the user to input their desired settings. @@ -791,13 +791,13 @@ It is also possible to use a DHCP reservation, but if you are going to do that, until [[ "${ipSettingsCorrect}" = True ]]; do # Ask for the IPv4 address - IPV4_ADDRESS=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" ${r} ${c} "${IPV4_ADDRESS}" 3>&1 1>&2 2>&3) || \ + IPV4_ADDRESS=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 address" --inputbox "Enter your desired IPv4 address" "${r}" "${c}" "${IPV4_ADDRESS}" 3>&1 1>&2 2>&3) || \ # Cancelling IPv4 settings window { ipSettingsCorrect=False; echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } printf " %b Your static IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}" # Ask for the gateway - IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" ${r} ${c} "${IPv4gw}" 3>&1 1>&2 2>&3) || \ + IPv4gw=$(whiptail --backtitle "Calibrating network interface" --title "IPv4 gateway (router)" --inputbox "Enter your desired IPv4 default gateway" "${r}" "${c}" "${IPv4gw}" 3>&1 1>&2 2>&3) || \ # Cancelling gateway settings window { ipSettingsCorrect=False; echo -e " ${COL_LIGHT_RED}Cancel was selected, exiting installer${COL_NC}"; exit 1; } printf " %b Your static IPv4 gateway: %s\\n" "${INFO}" "${IPv4gw}" @@ -805,7 +805,7 @@ It is also possible to use a DHCP reservation, but if you are going to do that, # Give the user a chance to review their settings before moving on if whiptail --backtitle "Calibrating network interface" --title "Static IP Address" --yesno "Are these settings correct? IP address: ${IPV4_ADDRESS} - Gateway: ${IPv4gw}" ${r} ${c}; then + Gateway: ${IPv4gw}" "${r}" "${c}"; then # After that's done, the loop ends and we move on ipSettingsCorrect=True else @@ -933,7 +933,7 @@ valid_ip() { # and set the new one to a dot (period) IFS='.' # Put the IP into an array - ip=(${ip}) + ip=("${ip}") # Restore the IFS to what it was IFS=${OIFS} ## Evaluate each octet by checking if it's less than or equal to 255 (the max for each octet) @@ -943,7 +943,7 @@ valid_ip() { stat=$? fi # Return the exit code - return ${stat} + return "${stat}" } # A function to choose the upstream DNS provider(s) @@ -973,7 +973,7 @@ setDNS() { # Restore the IFS to what it was IFS=${OIFS} # In a whiptail dialog, show the options - DNSchoices=$(whiptail --separate-output --menu "Select Upstream DNS Provider. To use your own, select Custom." ${r} ${c} 7 \ + DNSchoices=$(whiptail --separate-output --menu "Select Upstream DNS Provider. To use your own, select Custom." "${r}" "${c}" 7 \ "${DNSChooseOptions[@]}" 2>&1 >/dev/tty) || \ # exit if Cancel is selected { printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } @@ -1003,7 +1003,7 @@ setDNS() { fi # Dialog for the user to enter custom upstream servers - piholeDNS=$(whiptail --backtitle "Specify Upstream DNS Provider(s)" --inputbox "Enter your desired upstream DNS provider(s), separated by a comma.\\n\\nFor example '8.8.8.8, 8.8.4.4'" ${r} ${c} "${prePopulate}" 3>&1 1>&2 2>&3) || \ + piholeDNS=$(whiptail --backtitle "Specify Upstream DNS Provider(s)" --inputbox "Enter your desired upstream DNS provider(s), separated by a comma.\\n\\nFor example '8.8.8.8, 8.8.4.4'" "${r}" "${c}" "${prePopulate}" 3>&1 1>&2 2>&3) || \ { printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } # Clean user input and replace whitespace with comma. piholeDNS=$(sed 's/[, \t]\+/,/g' <<< "${piholeDNS}") @@ -1036,7 +1036,7 @@ setDNS() { # Otherwise, else # Show the settings - if (whiptail --backtitle "Specify Upstream DNS Provider(s)" --title "Upstream DNS Provider(s)" --yesno "Are these settings correct?\\n DNS Server 1: $PIHOLE_DNS_1\\n DNS Server 2: ${PIHOLE_DNS_2}" ${r} ${c}); then + if (whiptail --backtitle "Specify Upstream DNS Provider(s)" --title "Upstream DNS Provider(s)" --yesno "Are these settings correct?\\n DNS Server 1: $PIHOLE_DNS_1\\n DNS Server 2: ${PIHOLE_DNS_2}" "${r}" "${c}"); then # and break from the loop since the servers are valid DNSSettingsCorrect=True # Otherwise, @@ -1127,7 +1127,7 @@ setAdminFlag() { local WebChoices # Similar to the logging function, ask what the user wants - WebToggleCommand=(whiptail --separate-output --radiolist "Do you wish to install the web admin interface?" ${r} ${c} 6) + WebToggleCommand=(whiptail --separate-output --radiolist "Do you wish to install the web admin interface?" "${r}" "${c}" 6) # with the default being enabled WebChooseOptions=("On (Recommended)" "" on Off "" off) @@ -1248,11 +1248,11 @@ version_check_dnsmasq() { printf " it is from a previous Pi-hole install.\\n" printf " %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}" # so backup the original file - mv -f ${dnsmasq_conf} ${dnsmasq_conf_orig} + mv -f "${dnsmasq_conf}" "${dnsmasq_conf_orig}" printf "%b %b Backing up dnsmasq.conf to dnsmasq.conf.orig...\\n" "${OVER}" "${TICK}" printf " %b Restoring default dnsmasq.conf..." "${INFO}" # and replace it with the default - install -D -m 644 -T ${dnsmasq_original_config} ${dnsmasq_conf} + install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}" printf "%b %b Restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" # Otherwise, else @@ -1263,7 +1263,7 @@ version_check_dnsmasq() { # If a file cannot be found, printf " %b No dnsmasq.conf found... restoring default dnsmasq.conf..." "${INFO}" # restore the default one - install -D -m 644 -T ${dnsmasq_original_config} ${dnsmasq_conf} + install -D -m 644 -T "${dnsmasq_original_config}" "${dnsmasq_conf}" printf "%b %b No dnsmasq.conf found... restoring default dnsmasq.conf...\\n" "${OVER}" "${TICK}" fi @@ -1273,37 +1273,37 @@ version_check_dnsmasq() { install -d -m 755 "/etc/dnsmasq.d" fi # Copy the new Pi-hole DNS config file into the dnsmasq.d directory - install -D -m 644 -T ${dnsmasq_pihole_01_snippet} ${dnsmasq_pihole_01_location} + install -D -m 644 -T "${dnsmasq_pihole_01_snippet}" "${dnsmasq_pihole_01_location}" printf "%b %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf\\n" "${OVER}" "${TICK}" # Replace our placeholder values with the GLOBAL DNS variables that we populated earlier # First, swap in the interface to listen on - sed -i "s/@INT@/$PIHOLE_INTERFACE/" ${dnsmasq_pihole_01_location} + sed -i "s/@INT@/$PIHOLE_INTERFACE/" "${dnsmasq_pihole_01_location}" if [[ "${PIHOLE_DNS_1}" != "" ]]; then # Then swap in the primary DNS server - sed -i "s/@DNS1@/$PIHOLE_DNS_1/" ${dnsmasq_pihole_01_location} + sed -i "s/@DNS1@/$PIHOLE_DNS_1/" "${dnsmasq_pihole_01_location}" else # - sed -i '/^server=@DNS1@/d' ${dnsmasq_pihole_01_location} + sed -i '/^server=@DNS1@/d' "${dnsmasq_pihole_01_location}" fi if [[ "${PIHOLE_DNS_2}" != "" ]]; then # Then swap in the primary DNS server - sed -i "s/@DNS2@/$PIHOLE_DNS_2/" ${dnsmasq_pihole_01_location} + sed -i "s/@DNS2@/$PIHOLE_DNS_2/" "${dnsmasq_pihole_01_location}" else # - sed -i '/^server=@DNS2@/d' ${dnsmasq_pihole_01_location} + sed -i '/^server=@DNS2@/d' "${dnsmasq_pihole_01_location}" fi # - sed -i 's/^#conf-dir=\/etc\/dnsmasq.d$/conf-dir=\/etc\/dnsmasq.d/' ${dnsmasq_conf} + sed -i 's/^#conf-dir=\/etc\/dnsmasq.d$/conf-dir=\/etc\/dnsmasq.d/' "${dnsmasq_conf}" # If the user does not want to enable logging, if [[ "${QUERY_LOGGING}" == false ]] ; then # Disable it by commenting out the directive in the DNS config file - sed -i 's/^log-queries/#log-queries/' ${dnsmasq_pihole_01_location} + sed -i 's/^log-queries/#log-queries/' "${dnsmasq_pihole_01_location}" # Otherwise, else # enable it by uncommenting the directive in the DNS config file - sed -i 's/^#log-queries/log-queries/' ${dnsmasq_pihole_01_location} + sed -i 's/^#log-queries/log-queries/' "${dnsmasq_pihole_01_location}" fi } @@ -1647,7 +1647,7 @@ install_dependent_packages() { # Install Fedora/CentOS packages for i in "${argArray1[@]}"; do printf " %b Checking for %s..." "${INFO}" "${i}" - if ${PKG_MANAGER} -q list installed "${i}" &> /dev/null; then + if "${PKG_MANAGER}" -q list installed "${i}" &> /dev/null; then printf "%b %b Checking for %s" "${OVER}" "${TICK}" "${i}" else printf "%b %b Checking for %s (will be installed)" "${OVER}" "${INFO}" "${i}" @@ -1765,7 +1765,7 @@ configureFirewall() { # If a firewall is running, if firewall-cmd --state &> /dev/null; then # ask if the user wants to install Pi-hole's default firewall rules - whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" ${r} ${c} || \ + whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" "${r}" "${c}" || \ { printf " %b Not installing firewall rulesets.\\n" "${INFO}"; return 0; } printf " %b Configuring FirewallD for httpd and pihole-FTL\\n" "${TICK}" # Allow HTTP and DNS traffic @@ -1778,7 +1778,7 @@ configureFirewall() { # If chain Policy is not ACCEPT or last Rule is not ACCEPT # then check and insert our Rules above the DROP/REJECT Rule. if iptables -S INPUT | head -n1 | grep -qv '^-P.*ACCEPT$' || iptables -S INPUT | tail -n1 | grep -qv '^-\(A\|P\).*ACCEPT$'; then - whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" ${r} ${c} || \ + whiptail --title "Firewall in use" --yesno "We have detected a running firewall\\n\\nPi-hole currently requires HTTP and DNS port access.\\n\\n\\n\\nInstall Pi-hole default firewall rules?" "${r}" "${c}" || \ { printf " %b Not installing firewall rulesets.\\n" "${INFO}"; return 0; } printf " %b Installing new IPTables firewall rulesets\\n" "${TICK}" # Check chain first, otherwise a new rule will duplicate old ones @@ -1872,21 +1872,21 @@ installLogrotate() { # At some point in the future this list can be pruned, for now we'll need it to ensure updates don't break. # Refactoring of install script has changed the name of a couple of variables. Sort them out here. accountForRefactor() { - sed -i 's/piholeInterface/PIHOLE_INTERFACE/g' ${setupVars} - sed -i 's/IPv4_address/IPV4_ADDRESS/g' ${setupVars} - sed -i 's/IPv4addr/IPV4_ADDRESS/g' ${setupVars} - sed -i 's/IPv6_address/IPV6_ADDRESS/g' ${setupVars} - sed -i 's/piholeIPv6/IPV6_ADDRESS/g' ${setupVars} - sed -i 's/piholeDNS1/PIHOLE_DNS_1/g' ${setupVars} - sed -i 's/piholeDNS2/PIHOLE_DNS_2/g' ${setupVars} - sed -i 's/^INSTALL_WEB=/INSTALL_WEB_INTERFACE=/' ${setupVars} + sed -i 's/piholeInterface/PIHOLE_INTERFACE/g' "${setupVars}" + sed -i 's/IPv4_address/IPV4_ADDRESS/g' "${setupVars}" + sed -i 's/IPv4addr/IPV4_ADDRESS/g' "${setupVars}" + sed -i 's/IPv6_address/IPV6_ADDRESS/g' "${setupVars}" + sed -i 's/piholeIPv6/IPV6_ADDRESS/g' "${setupVars}" + sed -i 's/piholeDNS1/PIHOLE_DNS_1/g' "${setupVars}" + sed -i 's/piholeDNS2/PIHOLE_DNS_2/g' "${setupVars}" + sed -i 's/^INSTALL_WEB=/INSTALL_WEB_INTERFACE=/' "${setupVars}" # Add 'INSTALL_WEB_SERVER', if its not been applied already: https://github.com/pi-hole/pi-hole/pull/2115 if ! grep -q '^INSTALL_WEB_SERVER=' ${setupVars}; then local webserver_installed=false if grep -q '^INSTALL_WEB_INTERFACE=true' ${setupVars}; then webserver_installed=true fi - echo -e "INSTALL_WEB_SERVER=$webserver_installed" >> ${setupVars} + echo -e "INSTALL_WEB_SERVER=$webserver_installed" >> "${setupVars}" fi } @@ -1968,7 +1968,7 @@ checkSelinux() { # If it's enforcing, if [[ "${enforceMode}" == "Enforcing" ]]; then # Explain Pi-hole does not support it yet - whiptail --defaultno --title "SELinux Enforcing Detected" --yesno "SELinux is being ENFORCED on your system! \\n\\nPi-hole currently does not support SELinux, but you may still continue with the installation.\\n\\nNote: Web Admin will not be fully functional unless you set your policies correctly\\n\\nContinue installing Pi-hole?" ${r} ${c} || \ + whiptail --defaultno --title "SELinux Enforcing Detected" --yesno "SELinux is being ENFORCED on your system! \\n\\nPi-hole currently does not support SELinux, but you may still continue with the installation.\\n\\nNote: Web Admin will not be fully functional unless you set your policies correctly\\n\\nContinue installing Pi-hole?" "${r}" "${c}" || \ { printf "\\n %bSELinux Enforcing detected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } printf " %b Continuing installation with SELinux Enforcing\\n" "${INFO}" printf " %b Please refer to official SELinux documentation to create a custom policy\\n" "${INFO}" @@ -2007,7 +2007,7 @@ If you set a new IP address, you should restart the Pi. The install log is in /etc/pihole. -${additional}" ${r} ${c} +${additional}" "${r}" "${c}" } update_dialogs() { @@ -2028,7 +2028,7 @@ update_dialogs() { opt2b="This will reset your Pi-hole and allow you to enter new settings." # Display the information to the user - UpdateCmd=$(whiptail --title "Existing Install Detected!" --menu "\\n\\nWe have detected an existing install.\\n\\nPlease choose from the following options: \\n($strAdd)" ${r} ${c} 2 \ + UpdateCmd=$(whiptail --title "Existing Install Detected!" --menu "\\n\\nWe have detected an existing install.\\n\\nPlease choose from the following options: \\n($strAdd)" "${r}" "${c}" 2 \ "${opt1a}" "${opt1b}" \ "${opt2a}" "${opt2b}" 3>&2 2>&1 1>&3) || \ { printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } @@ -2556,7 +2556,7 @@ main() { installDefaultBlocklists # Source ${setupVars} to use predefined user variables in the functions - source ${setupVars} + source "${setupVars}" # Get the privacy level if it exists (default is 0) if [[ -f "${PI_HOLE_CONFIG_DIR}/pihole-FTL.conf" ]]; then @@ -2613,7 +2613,7 @@ main() { pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8) # shellcheck disable=SC1091 . /opt/pihole/webpage.sh - echo "WEBPASSWORD=$(HashPassword ${pw})" >> ${setupVars} + echo "WEBPASSWORD=$(HashPassword "${pw}")" >> "${setupVars}" fi fi From 69081a91757590f6a7d1ef157c786be418d2a7e8 Mon Sep 17 00:00:00 2001 From: David Haguenauer Date: Fri, 10 May 2019 10:37:38 -0400 Subject: [PATCH 090/207] Drop indirection from install_dependent_packages Previously, install_dependent_packages would receive an array variable name as its single parameter, and would use variable indirection to access it; this change simplifies that function so that it instead receives the expanded array. Signed-off-by: David Haguenauer --- automated install/basic-install.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 35d3305c..a5a78102 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1616,7 +1616,6 @@ install_dependent_packages() { # Install packages passed in via argument array # No spinner - conflicts with set -e - declare -a argArray1=("${!1}") declare -a installArray # Debian based package install - debconf will download the entire package list @@ -1626,7 +1625,7 @@ install_dependent_packages() { # installed by us, and remove only the installed packages, and not the entire list. if is_command debconf-apt-progress ; then # For each package, - for i in "${argArray1[@]}"; do + for i in "$@"; do printf " %b Checking for %s..." "${INFO}" "${i}" if dpkg-query -W -f='${Status}' "${i}" 2>/dev/null | grep "ok installed" &> /dev/null; then printf "%b %b Checking for %s\\n" "${OVER}" "${TICK}" "${i}" @@ -1645,7 +1644,7 @@ install_dependent_packages() { fi # Install Fedora/CentOS packages - for i in "${argArray1[@]}"; do + for i in "$@"; do printf " %b Checking for %s..." "${INFO}" "${i}" if "${PKG_MANAGER}" -q list installed "${i}" &> /dev/null; then printf "%b %b Checking for %s" "${OVER}" "${TICK}" "${i}" @@ -2525,7 +2524,7 @@ main() { notify_package_updates_available # Install packages used by this installation script - install_dependent_packages INSTALLER_DEPS[@] + install_dependent_packages "${INSTALLER_DEPS[@]}" # Check if SELinux is Enforcing checkSelinux @@ -2576,7 +2575,7 @@ main() { dep_install_list+=("${PIHOLE_WEB_DEPS[@]}") fi - install_dependent_packages dep_install_list[@] + install_dependent_packages "${dep_install_list[@]}" unset dep_install_list # On some systems, lighttpd is not enabled on first install. We need to enable it here if the user From be3a21ae0770371c0614f1923f02ac504c53df0a Mon Sep 17 00:00:00 2001 From: David Haguenauer Date: Mon, 13 May 2019 09:13:23 -0400 Subject: [PATCH 091/207] Put quotes at ends of strings Signed-off-by: David Haguenauer --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a5a78102..bb000808 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -233,7 +233,7 @@ if is_command apt-get ; then phpVer="php$phpInsMajor.$phpInsMinor" fi # We also need the correct version for `php-sqlite` (which differs across distros) - if "${PKG_MANAGER}" install --dry-run "${phpVer}"-sqlite3 > /dev/null 2>&1; then + if "${PKG_MANAGER}" install --dry-run "${phpVer}-sqlite3" > /dev/null 2>&1; then phpSqlite="sqlite3" else phpSqlite="sqlite" @@ -245,7 +245,7 @@ if is_command apt-get ; then PIHOLE_DEPS=(cron curl dnsutils iputils-ping lsof netcat psmisc sudo unzip wget idn2 sqlite3 libcap2-bin dns-root-data resolvconf libcap2) # The Web dashboard has some that also need to be installed # It's useful to separate the two since our repos are also setup as "Core" code and "Web" code - PIHOLE_WEB_DEPS=(lighttpd "${phpVer}"-common "${phpVer}"-cgi "${phpVer}-${phpSqlite}") + PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-${phpSqlite}") # The Web server user, LIGHTTPD_USER="www-data" # group, From 03d93aa19a61cbe3e23dea47c3bdd9cd2c66287a Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 20 May 2019 20:58:57 -0700 Subject: [PATCH 092/207] Update debug script with gravity DB changes Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 67 +++++++++++++++++---------------- 1 file changed, 35 insertions(+), 32 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 1010f26c..3a5c482f 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -89,16 +89,12 @@ PIHOLE_WILDCARD_CONFIG_FILE="${DNSMASQ_D_DIRECTORY}/03-wildcard.conf" WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" #WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" -PIHOLE_DEFAULT_AD_LISTS="${PIHOLE_DIRECTORY}/adlists.default" -PIHOLE_USER_DEFINED_AD_LISTS="${PIHOLE_DIRECTORY}/adlists.list" -PIHOLE_BLACKLIST_FILE="${PIHOLE_DIRECTORY}/blacklist.txt" -PIHOLE_BLOCKLIST_FILE="${PIHOLE_DIRECTORY}/gravity.list" PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_SETUP_VARS_FILE="${PIHOLE_DIRECTORY}/setupVars.conf" -PIHOLE_WHITELIST_FILE="${PIHOLE_DIRECTORY}/whitelist.txt" +PIHOLE_GRAVITY_DB_FILE="${PIHOLE_DIRECTORY}/gravity.db" PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole" PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE" @@ -142,16 +138,11 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}" "${PIHOLE_DHCP_CONFIG_FILE}" "${PIHOLE_WILDCARD_CONFIG_FILE}" "${WEB_SERVER_CONFIG_FILE}" -"${PIHOLE_DEFAULT_AD_LISTS}" -"${PIHOLE_USER_DEFINED_AD_LISTS}" -"${PIHOLE_BLACKLIST_FILE}" -"${PIHOLE_BLOCKLIST_FILE}" "${PIHOLE_INSTALL_LOG_FILE}" "${PIHOLE_RAW_BLOCKLIST_FILES}" "${PIHOLE_LOCAL_HOSTS_FILE}" "${PIHOLE_LOGROTATE_FILE}" "${PIHOLE_SETUP_VARS_FILE}" -"${PIHOLE_WHITELIST_FILE}" "${PIHOLE_COMMAND}" "${PIHOLE_COLTABLE_FILE}" "${FTL_PID}" @@ -793,7 +784,7 @@ dig_at() { # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(shuf -n 1 "${PIHOLE_BLOCKLIST_FILE}") + random_url=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1") # First, do a dig on localhost to see if Pi-hole can use itself to block a domain if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then @@ -975,8 +966,7 @@ list_files_in_dir() { if [[ -d "${dir_to_parse}/${each_file}" ]]; then # If it's a directoy, do nothing : - elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_BLOCKLIST_FILE}" ]] || \ - [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \ + elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_RAW_BLOCKLIST_FILES}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_INSTALL_LOG_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_SETUP_VARS_FILE}" ]] || \ @@ -1061,31 +1051,43 @@ head_tail_log() { IFS="$OLD_IFS" } -analyze_gravity_list() { - echo_current_diagnostic "Gravity list" - local head_line - local tail_line - # Put the current Internal Field Separator into another variable so it can be restored later +show_adlists() { + echo_current_diagnostic "Adlists" + OLD_IFS="$IFS" - # Get the lines that are in the file(s) and store them in an array for parsing later IFS=$'\r\n' + local adlists=() + mapfile -t adlists < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT address FROM vw_adlists") + + for line in "${adlists[@]}"; do + log_write " ${line}" + done + + IFS="$OLD_IFS" +} + +analyze_gravity_list() { + echo_current_diagnostic "Gravity List and Database" + local gravity_permissions - gravity_permissions=$(ls -ld "${PIHOLE_BLOCKLIST_FILE}") + gravity_permissions=$(ls -ld "${PIHOLE_GRAVITY_DB_FILE}") log_write "${COL_GREEN}${gravity_permissions}${COL_NC}" - local gravity_head=() - mapfile -t gravity_head < <(head -n 4 ${PIHOLE_BLOCKLIST_FILE}) - log_write " ${COL_CYAN}-----head of $(basename ${PIHOLE_BLOCKLIST_FILE})------${COL_NC}" - for head_line in "${gravity_head[@]}"; do - log_write " ${head_line}" + + local gravity_size + gravity_size=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT COUNT(*) FROM vw_gravity") + log_write " Size: ${COL_CYAN}${gravity_size}${COL_NC} entries" + + OLD_IFS="$IFS" + IFS=$'\r\n' + local gravity_sample=() + mapfile -t gravity_sample < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") + log_write " ${COL_CYAN}----- First 10 Domains -----${COL_NC}" + + for line in "${gravity_sample[@]}"; do + log_write " ${line}" done + log_write "" - local gravity_tail=() - mapfile -t gravity_tail < <(tail -n 4 ${PIHOLE_BLOCKLIST_FILE}) - log_write " ${COL_CYAN}-----tail of $(basename ${PIHOLE_BLOCKLIST_FILE})------${COL_NC}" - for tail_line in "${gravity_tail[@]}"; do - log_write " ${tail_line}" - done - # Set the IFS back to what it was IFS="$OLD_IFS" } @@ -1236,6 +1238,7 @@ process_status parse_setup_vars check_x_headers analyze_gravity_list +show_adlists show_content_of_pihole_files parse_locale analyze_pihole_log From 3f05efd60f4e4ff5630621d3a1d50fd81e1e1807 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 20 May 2019 21:02:31 -0700 Subject: [PATCH 093/207] Add extra newline Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 3a5c482f..82660d61 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1076,6 +1076,7 @@ analyze_gravity_list() { local gravity_size gravity_size=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT COUNT(*) FROM vw_gravity") log_write " Size: ${COL_CYAN}${gravity_size}${COL_NC} entries" + log_write "" OLD_IFS="$IFS" IFS=$'\r\n' From 807ce0af4ed7d59830c1ae26eaffb6757c320699 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 20 May 2019 21:15:22 -0700 Subject: [PATCH 094/207] Show whitelist, blacklist, and regexlist details Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 82660d61..c491b1c6 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1051,21 +1051,40 @@ head_tail_log() { IFS="$OLD_IFS" } -show_adlists() { - echo_current_diagnostic "Adlists" +show_db_entries() { + local title="${1}" + local query="${2}" + + echo_current_diagnostic "${title}" OLD_IFS="$IFS" IFS=$'\r\n' - local adlists=() - mapfile -t adlists < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT address FROM vw_adlists") + local entries=() + mapfile -t entries < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" -cmd ".headers on" "${query}") - for line in "${adlists[@]}"; do + for line in "${entries[@]}"; do log_write " ${line}" done IFS="$OLD_IFS" } +show_adlists() { + show_db_entries "Adlists" "SELECT * FROM adlists" +} + +show_whitelist() { + show_db_entries "Whitelist" "SELECT * FROM whitelist" +} + +show_blacklist() { + show_db_entries "Blacklist" "SELECT * FROM blacklist" +} + +show_regexlist() { + show_db_entries "Regexlist" "SELECT * FROM regex" +} + analyze_gravity_list() { echo_current_diagnostic "Gravity List and Database" @@ -1240,6 +1259,9 @@ parse_setup_vars check_x_headers analyze_gravity_list show_adlists +show_whitelist +show_blacklist +show_regexlist show_content_of_pihole_files parse_locale analyze_pihole_log From 7b5fc60e003263b58fc7f1ebf8ce3bc3f6b74796 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 20 May 2019 21:20:38 -0700 Subject: [PATCH 095/207] Improve table formatting Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c491b1c6..e56d1f94 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1060,7 +1060,7 @@ show_db_entries() { OLD_IFS="$IFS" IFS=$'\r\n' local entries=() - mapfile -t entries < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" -cmd ".headers on" "${query}") + mapfile -t entries < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" -cmd ".headers on" -cmd ".mode column" "${query}") for line in "${entries[@]}"; do log_write " ${line}" From a3e1473ac10411e2fc72952bb2fa7983393b3b17 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 20 May 2019 21:33:09 -0700 Subject: [PATCH 096/207] Set explicit column widths to prevent text from getting cut off Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index e56d1f94..816652e0 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1054,13 +1054,20 @@ head_tail_log() { show_db_entries() { local title="${1}" local query="${2}" + local widths="${3}" echo_current_diagnostic "${title}" OLD_IFS="$IFS" IFS=$'\r\n' local entries=() - mapfile -t entries < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" -cmd ".headers on" -cmd ".mode column" "${query}") + mapfile -t entries < <(\ + sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \ + -cmd ".headers on" \ + -cmd ".mode column" \ + -cmd ".width ${widths}" \ + "${query}"\ + ) for line in "${entries[@]}"; do log_write " ${line}" @@ -1070,19 +1077,19 @@ show_db_entries() { } show_adlists() { - show_db_entries "Adlists" "SELECT * FROM adlists" + show_db_entries "Adlists" "SELECT * FROM adlists" "2 100 7 10 13 50" } show_whitelist() { - show_db_entries "Whitelist" "SELECT * FROM whitelist" + show_db_entries "Whitelist" "SELECT * FROM whitelist" "2 100 7 10 13 50" } show_blacklist() { - show_db_entries "Blacklist" "SELECT * FROM blacklist" + show_db_entries "Blacklist" "SELECT * FROM blacklist" "2 100 7 10 13 50" } show_regexlist() { - show_db_entries "Regexlist" "SELECT * FROM regex" + show_db_entries "Regexlist" "SELECT * FROM regex" "2 100 7 10 13 50" } analyze_gravity_list() { From 5796054305e4b6842d95f8f1259df35862a34ba4 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 20 May 2019 21:59:18 -0700 Subject: [PATCH 097/207] Increase ID column width to 4 Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 816652e0..07a11ff2 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1077,19 +1077,19 @@ show_db_entries() { } show_adlists() { - show_db_entries "Adlists" "SELECT * FROM adlists" "2 100 7 10 13 50" + show_db_entries "Adlists" "SELECT * FROM adlists" "4 100 7 10 13 50" } show_whitelist() { - show_db_entries "Whitelist" "SELECT * FROM whitelist" "2 100 7 10 13 50" + show_db_entries "Whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50" } show_blacklist() { - show_db_entries "Blacklist" "SELECT * FROM blacklist" "2 100 7 10 13 50" + show_db_entries "Blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50" } show_regexlist() { - show_db_entries "Regexlist" "SELECT * FROM regex" "2 100 7 10 13 50" + show_db_entries "Regexlist" "SELECT * FROM regex" "4 100 7 10 13 50" } analyze_gravity_list() { From bfb99c361c6db92a728760ba34edb63ffca6006c Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Tue, 21 May 2019 17:12:47 -0700 Subject: [PATCH 098/207] Note that the gravity size does not include the blacklist entries Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 07a11ff2..b31bbdc5 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1101,7 +1101,7 @@ analyze_gravity_list() { local gravity_size gravity_size=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT COUNT(*) FROM vw_gravity") - log_write " Size: ${COL_CYAN}${gravity_size}${COL_NC} entries" + log_write " Size (excluding blacklist): ${COL_CYAN}${gravity_size}${COL_NC} entries" log_write "" OLD_IFS="$IFS" From 69dba022c4cc9872fe3ea64b4d6ddcbbf438712f Mon Sep 17 00:00:00 2001 From: Chris Crocker-White Date: Mon, 27 May 2019 10:27:28 -0700 Subject: [PATCH 099/207] Handle an empty local.list Handle the case of an empty local.list file which would otherwise prevent the system from starting Change-type: patch Signed-off-by: Chris Crocker-White --- gravity.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gravity.sh b/gravity.sh index b41e8893..1fe2b4e3 100755 --- a/gravity.sh +++ b/gravity.sh @@ -184,8 +184,8 @@ migrate_to_database() { gravity_CheckDNSResolutionAvailable() { local lookupDomain="pi.hole" - # Determine if $localList does not exist - if [[ ! -e "${localList}" ]]; then + # Determine if $localList does not exist, and ensure it is not empty + if [[ ! -e "${localList}" ]] || [[ -s "${localList}" ]]; then lookupDomain="raw.githubusercontent.com" fi From f6213d4f4dfb2aa32dabf07619d9739116308bbd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 May 2019 15:26:27 +0200 Subject: [PATCH 100/207] Use last PID in case pidof returns multiple PIDs for pihole-FTL Signed-off-by: DL6ER --- advanced/Templates/pihole-FTL.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 9eb183ed..7c7e533e 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -13,7 +13,7 @@ FTLUSER=pihole PIDFILE=/var/run/pihole-FTL.pid get_pid() { - pidof "pihole-FTL" + pidof "pihole-FTL" | awk '{print $(NF)}' } is_running() { From c5df104a6653ae08b5d226de345a9f2cd53cf26b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 May 2019 16:41:37 +0200 Subject: [PATCH 101/207] Add dhcp-ignore-names option when enabling DHCP service. We currently remove anything that starts with "dhcp-" to have a clean configuration and removed these lines without noticing when enabling the DHCP server. Signed-off-by: DL6ER --- advanced/01-pihole.conf | 5 ----- advanced/Scripts/webpage.sh | 8 ++++++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf index 40a117fe..cd74e186 100644 --- a/advanced/01-pihole.conf +++ b/advanced/01-pihole.conf @@ -41,8 +41,3 @@ log-facility=/var/log/pihole.log local-ttl=2 log-async - -# If a DHCP client claims that its name is "wpad", ignore that. -# This fixes a security hole. see CERT Vulnerability VU#598349 -dhcp-name-match=set:wpad-ignore,wpad -dhcp-ignore-names=tag:wpad-ignore diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index ea699efa..583579b6 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -366,6 +366,14 @@ EnableDHCP() { delete_dnsmasq_setting "dhcp-" delete_dnsmasq_setting "quiet-dhcp" + # If a DHCP client claims that its name is "wpad", ignore that. + # This fixes a security hole. see CERT Vulnerability VU#598349 + # We also ignore "localhost" as Windows behaves strangely if a + # device claims this host name + add_dnsmasq_setting "dhcp-name-match=set:hostname-ignore,wpad +dhcp-name-match=set:hostname-ignore,localhost +dhcp-ignore-names=tag:hostname-ignore" + ProcessDHCPSettings RestartDNS From c3023fe68182c7c9adf2c39e86576cedec1bf942 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 May 2019 21:23:15 +0200 Subject: [PATCH 102/207] Add new "pihole arpflush" command to flush both the ARP cache as well as the network table in pihole-FTL.db Signed-off-by: DL6ER --- advanced/Scripts/piholeARPTable.sh | 65 ++++++++++++++++++++++++++++++ pihole | 10 ++++- 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100755 advanced/Scripts/piholeARPTable.sh diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh new file mode 100755 index 00000000..6af96d91 --- /dev/null +++ b/advanced/Scripts/piholeARPTable.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090 + +# Pi-hole: A black hole for Internet advertisements +# (c) 2019 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# ARP table interaction +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +coltable="/opt/pihole/COL_TABLE" +if [[ -f ${coltable} ]]; then + source ${coltable} +fi + +# Determine database location +# Obtain DBFILE=... setting from pihole-FTL.db +# Constructed to return nothing when +# a) the setting is not present in the config file, or +# b) the setting is commented out (e.g. "#DBFILE=...") +FTLconf="/etc/pihole/pihole-FTL.conf" +if [ -e "$FTLconf" ]; then + DBFILE="$(sed -n -e 's/^\s*DBFILE\s*=\s*//p' ${FTLconf})" +fi +# Test for empty string. Use standard path in this case. +if [ -z "$DBFILE" ]; then + DBFILE="/etc/pihole/pihole-FTL.db" +fi + + +flushARP(){ + local output + if [[ "$@" != *"quiet"* ]]; then + echo -ne " ${INFO} Flushing network table ..." + fi + + # Flush ARP cache to avoid re-adding of dead entries + if ! output=$(ip neigh flush all 2>&1); then + echo -e "${OVER} ${CROSS} Failed to clear ARP cache" + echo " Output: ${output}" + return 1 + fi + + if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network;" 2>&1); then + echo -e "${OVER} ${CROSS} Failed to truncate network table" + echo " Database location: ${DBFILE}" + echo " Output: ${output}" + return 1 + fi + + if [[ "$@" != *"quiet"* ]]; then + echo -e "${OVER} ${TICK} Flushed network table" + fi +} + +args=("$@") + +case "${args[0]}" in + "arpflush" ) flushARP;; +esac + +shift + diff --git a/pihole b/pihole index 71c286c6..d837f873 100755 --- a/pihole +++ b/pihole @@ -54,6 +54,11 @@ flushFunc() { exit 0 } +arpFunc() { + "${PI_HOLE_SCRIPT_DIR}"/piholeARPTable.sh "$@" + exit 0 +} + updatePiholeFunc() { shift "${PI_HOLE_SCRIPT_DIR}"/update.sh "$@" @@ -430,8 +435,8 @@ fi case "${1}" in "-w" | "whitelist" ) listFunc "$@";; "-b" | "blacklist" ) listFunc "$@";; - "--wild" | "wildcard" ) listFunc "$@";; - "--regex" | "regex" ) listFunc "$@";; + "--wild" | "wildcard" ) listFunc "$@";; + "--regex" | "regex" ) listFunc "$@";; "-d" | "debug" ) debugFunc "$@";; "-f" | "flush" ) flushFunc "$@";; "-up" | "updatePihole" ) updatePiholeFunc "$@";; @@ -452,5 +457,6 @@ case "${1}" in "checkout" ) piholeCheckoutFunc "$@";; "tricorder" ) tricorderFunc;; "updatechecker" ) updateCheckFunc "$@";; + "arpflush" ) arpFunc "$@";; * ) helpFunc;; esac From 285e6fe090069ffd38e6679b38256a9632e2f9e8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 May 2019 21:32:35 +0200 Subject: [PATCH 103/207] Address lint complaints Signed-off-by: DL6ER --- advanced/Scripts/piholeARPTable.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index 6af96d91..dab730f4 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -32,7 +32,7 @@ fi flushARP(){ local output - if [[ "$@" != *"quiet"* ]]; then + if [[ "${args[1]}" != *"quiet"* ]]; then echo -ne " ${INFO} Flushing network table ..." fi @@ -43,6 +43,7 @@ flushARP(){ return 1 fi + # Truncate network table in pihole-FTL.db if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network;" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network table" echo " Database location: ${DBFILE}" @@ -50,7 +51,7 @@ flushARP(){ return 1 fi - if [[ "$@" != *"quiet"* ]]; then + if [[ "${args[1]}" != *"quiet"* ]]; then echo -e "${OVER} ${TICK} Flushed network table" fi } From 5060605626950007aa0fc35153356ea49c96a187 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 30 May 2019 21:44:47 +0200 Subject: [PATCH 104/207] Print table name before entering the loop for the sake of simplicity Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index d2e16e79..88ec2173 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -103,7 +103,7 @@ if [[ -n "${str:-}" ]]; then fi scanDatabaseTable() { - local domain table type querystr result table_prev + local domain table type querystr result domain="$(printf "%q" "${1}")" table="${2}" type="${3:-}" @@ -126,22 +126,20 @@ scanDatabaseTable() { # Mark domain as having been white-/blacklist matched (global variable) wbMatch=true - # Loop through each result + + # Print table name + if [[ ! -z "${result}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" + fi + + # Loop over results and print them mapfile -t results <<< "${result}" for result in "${results[@]}"; do if [[ -n "${blockpage}" ]]; then echo "π ${result}" exit 0 - elif [[ -n "${exact}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" - else - # Only print table name once - if [[ ! "${table}" == "${table_prev:-}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" - table_prev="${table}" - fi - echo " ${result}" fi + echo " ${result}" done } From 44f8fcb600cc78f7b4dd8c6c9c8bba73c64c718c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 31 May 2019 08:39:18 +0200 Subject: [PATCH 105/207] We can print the table name without any extra check as we already returned early in case there are no results. Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 88ec2173..9cb298df 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -128,9 +128,7 @@ scanDatabaseTable() { wbMatch=true # Print table name - if [[ ! -z "${result}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" - fi + echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" # Loop over results and print them mapfile -t results <<< "${result}" From 9ddce880920752cece01b48ad25caaeffe9275b8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 31 May 2019 08:42:22 +0200 Subject: [PATCH 106/207] Review comments Signed-off-by: DL6ER --- advanced/Scripts/piholeARPTable.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index dab730f4..10bee0df 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -32,7 +32,7 @@ fi flushARP(){ local output - if [[ "${args[1]}" != *"quiet"* ]]; then + if [[ "${args[1]}" != "quiet" ]]; then echo -ne " ${INFO} Flushing network table ..." fi @@ -44,14 +44,14 @@ flushARP(){ fi # Truncate network table in pihole-FTL.db - if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network;" 2>&1); then + if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network table" echo " Database location: ${DBFILE}" echo " Output: ${output}" return 1 fi - if [[ "${args[1]}" != *"quiet"* ]]; then + if [[ "${args[1]}" != "quiet" ]]; then echo -e "${OVER} ${TICK} Flushed network table" fi } @@ -61,6 +61,3 @@ args=("$@") case "${args[0]}" in "arpflush" ) flushARP;; esac - -shift - From ae98fde32141154e7384bf38a3893ecf162d94ca Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 31 May 2019 09:18:12 +0200 Subject: [PATCH 107/207] Try to obtain PID from PIDFILE. If this fails (file does not exist or is empty), fall back to using pidof + awk Signed-off-by: DL6ER --- advanced/Templates/pihole-FTL.service | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 7c7e533e..8a4c7ce6 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -13,6 +13,13 @@ FTLUSER=pihole PIDFILE=/var/run/pihole-FTL.pid get_pid() { + # First, try to obtain PID from PIDFILE + if [ -s "${PIDFILE}" ]; then + cat "${PIDFILE}" + return + fi + + # If the PIDFILE is empty or not available, obtain the PID using pidof pidof "pihole-FTL" | awk '{print $(NF)}' } From a09f92f9cc664cc21381ab2d651c7803e7e361a0 Mon Sep 17 00:00:00 2001 From: Dan Schaper Date: Fri, 31 May 2019 22:12:54 -0700 Subject: [PATCH 108/207] Create FUNDING.yml Signed-off-by: Dan Schaper --- .github/FUNDING.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..3a75dc12 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,4 @@ +# These are supported funding model platforms + +patreon: pihole +custom: https://pi-hole.net/donate From 54bfaa28c1ac41fc8839c935003a1900320195c2 Mon Sep 17 00:00:00 2001 From: MMotti Date: Mon, 3 Jun 2019 19:23:27 +0100 Subject: [PATCH 109/207] Fix for regexp queries through pihole -q Taking inspiration from: https://github.com/pi-hole/pi-hole/pull/2346 We will use awk to iterate through each regexp and print any that match the domain query. Signed-off-by: MMotti --- advanced/Scripts/query.sh | 64 +++++++++++++++------------------------ 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 9cb298df..840c03da 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -12,7 +12,7 @@ # Globals piholeDir="/etc/pihole" gravityDBfile="${piholeDir}/gravity.db" -wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf" +regexlist="/etc/pihole/regex.list" options="$*" adlist="" all="" @@ -23,27 +23,11 @@ matchType="match" colfile="/opt/pihole/COL_TABLE" source "${colfile}" -# Print each subdomain -# e.g: foo.bar.baz.com = "foo.bar.baz.com bar.baz.com baz.com com" -processWildcards() { - IFS="." read -r -a array <<< "${1}" - for (( i=${#array[@]}-1; i>=0; i-- )); do - ar="" - for (( j=${#array[@]}-1; j>${#array[@]}-i-2; j-- )); do - if [[ $j == $((${#array[@]}-1)) ]]; then - ar="${array[$j]}" - else - ar="${array[$j]}.${ar}" - fi - done - echo "${ar}" - done -} - +# Scan an array of files for matching strings # Scan an array of files for matching strings scanList(){ # Escape full stops - local domain="${1//./\\.}" lists="${2}" type="${3:-}" + local domain="${1}" esc_domain="${1//./\\.}" lists="${2}" type="${3:-}" # Prevent grep from printing file path cd "$piholeDir" || exit 1 @@ -54,9 +38,9 @@ scanList(){ # /dev/null forces filename to be printed when only one list has been generated # shellcheck disable=SC2086 case "${type}" in - "exact" ) grep -i -E -l "(^|(?/dev/null;; - "wc" ) grep -i -o -m 1 "/${domain}/" ${lists} 2>/dev/null;; - * ) grep -i "${domain}" ${lists} /dev/null 2>/dev/null;; + "exact" ) grep -i -E -l "(^|(?/dev/null;; + "rx" ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' ${lists} <(echo "$domain") 2>/dev/null;; + * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; esac } @@ -145,24 +129,26 @@ scanDatabaseTable() { scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" -# Scan Wildcards -if [[ -e "${wildcardlist}" ]]; then - # Determine all subdomains, domain and TLDs - mapfile -t wildcards <<< "$(processWildcards "${domainQuery}")" - for match in "${wildcards[@]}"; do - # Search wildcard list for matches - mapfile -t results <<< "$(scanList "${match}" "${wildcardlist}" "wc")" - if [[ -n "${results[*]}" ]]; then - if [[ -z "${wcMatch:-}" ]] && [[ -z "${blockpage}" ]]; then - wcMatch=true - echo " ${matchType^} found in ${COL_BOLD}Wildcards${COL_NC}:" - fi - case "${blockpage}" in - true ) echo "π ${wildcardlist##*/}"; exit 0;; - * ) echo " *.${match}";; - esac +# Scan Regex +if [[ -e "${regexlist}" ]]; then + # Return portion(s) of string that is found in the regex list + mapfile -t results <<< "$(scanList "${domainQuery}" "${regexlist}" "rx")" + + if [[ -n "${results[*]}" ]]; then + # A result is found + str="Phrase ${matchType}ed within ${COL_BOLD}regex list${COL_NC}" + result="${COL_BOLD}$(printf '%s\n' ${results[*]})${COL_NC}" + + if [[ -z "${blockpage}" ]]; then + wcMatch=true + echo " $str" fi - done + + case "${blockpage}" in + true ) echo "π ${regexlist##*/}"; exit 0;; + * ) awk '{print " "$0}' <<< "${result}";; + esac + fi fi # Get version sorted *.domains filenames (without dir path) From 97df6d7415bfc2daf10b401a8c8712a47af0ab2d Mon Sep 17 00:00:00 2001 From: MMotti Date: Mon, 3 Jun 2019 19:55:29 +0100 Subject: [PATCH 110/207] Stickler fix Signed-off-by: MMotti --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 840c03da..827f97a3 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -137,7 +137,7 @@ if [[ -e "${regexlist}" ]]; then if [[ -n "${results[*]}" ]]; then # A result is found str="Phrase ${matchType}ed within ${COL_BOLD}regex list${COL_NC}" - result="${COL_BOLD}$(printf '%s\n' ${results[*]})${COL_NC}" + result="${COL_BOLD}$(IFS=$'\n'; echo "${results[*]}")${COL_NC}" if [[ -z "${blockpage}" ]]; then wcMatch=true From 09532638d5d6cbbf82aa44ec0e04a67e64402905 Mon Sep 17 00:00:00 2001 From: MMotti Date: Mon, 3 Jun 2019 23:59:58 +0100 Subject: [PATCH 111/207] Read from DB instead of regex.list Signed-off-by: MMotti --- advanced/Scripts/query.sh | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 827f97a3..a4cf02b1 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -39,7 +39,7 @@ scanList(){ # shellcheck disable=SC2086 case "${type}" in "exact" ) grep -i -E -l "(^|(?/dev/null;; - "rx" ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' ${lists} <(echo "$domain") 2>/dev/null;; + "rx" ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' <(echo "$lists") <(echo "$domain") 2>/dev/null;; * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; esac } @@ -97,8 +97,9 @@ scanDatabaseTable() { # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. case "${type}" in - "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; - * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; + "retrievetable" ) querystr="SELECT domain FROM vw_${table}";; + * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; esac # Send prepared query to gravity database @@ -108,6 +109,13 @@ scanDatabaseTable() { return fi + # If we are only retrieving the table + # Just output and return + if [[ "${type}" == "retrievetable" ]]; then + echo "${result[*]}" + return + fi + # Mark domain as having been white-/blacklist matched (global variable) wbMatch=true @@ -129,14 +137,21 @@ scanDatabaseTable() { scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" -# Scan Regex -if [[ -e "${regexlist}" ]]; then +# Scan Regex table +regexlist=$(scanDatabaseTable "" "regex" "retrievetable") + +if [[ -n "${regexlist}" ]]; then # Return portion(s) of string that is found in the regex list mapfile -t results <<< "$(scanList "${domainQuery}" "${regexlist}" "rx")" - if [[ -n "${results[*]}" ]]; then - # A result is found - str="Phrase ${matchType}ed within ${COL_BOLD}regex list${COL_NC}" + # If a result is found + if [[ -n "${results[*]}" ]]; then + # Count the matches + regexCount=${#results[@]} + # Determine plural string + [[ $regexCount -gt 1 ]] && plu="es" + # Form output strings + str="${COL_BOLD}${regexCount}${COL_NC} ${matchType}${plu:-} found in ${COL_BOLD}regex${COL_NC} table" result="${COL_BOLD}$(IFS=$'\n'; echo "${results[*]}")${COL_NC}" if [[ -z "${blockpage}" ]]; then From b49c702f331c5bfecc2b4622039740f8b32aa247 Mon Sep 17 00:00:00 2001 From: MMotti Date: Tue, 4 Jun 2019 00:03:37 +0100 Subject: [PATCH 112/207] Consistency tweak Signed-off-by: MMotti --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index a4cf02b1..f4aed05b 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -39,7 +39,7 @@ scanList(){ # shellcheck disable=SC2086 case "${type}" in "exact" ) grep -i -E -l "(^|(?/dev/null;; - "rx" ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' <(echo "$lists") <(echo "$domain") 2>/dev/null;; + "rx" ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' <(echo "${lists}") <(echo "${domain}") 2>/dev/null;; * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; esac } From 7613e94ef6f987d12203dc13e26b5637a5e31d48 Mon Sep 17 00:00:00 2001 From: MMotti Date: Tue, 4 Jun 2019 06:06:17 +0100 Subject: [PATCH 113/207] Minor tweaks Mainly for consistency Signed-off-by: MMotti --- advanced/Scripts/query.sh | 44 +++++++++++++++++---------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index f4aed05b..5b8baa63 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -12,7 +12,6 @@ # Globals piholeDir="/etc/pihole" gravityDBfile="${piholeDir}/gravity.db" -regexlist="/etc/pihole/regex.list" options="$*" adlist="" all="" @@ -23,7 +22,6 @@ matchType="match" colfile="/opt/pihole/COL_TABLE" source "${colfile}" -# Scan an array of files for matching strings # Scan an array of files for matching strings scanList(){ # Escape full stops @@ -39,7 +37,12 @@ scanList(){ # shellcheck disable=SC2086 case "${type}" in "exact" ) grep -i -E -l "(^|(?/dev/null;; - "rx" ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' <(echo "${lists}") <(echo "${domain}") 2>/dev/null;; + # Create array of regexps + # Iterate through each regexp and check whether it matches the domainQuery + # If it does, print the matching regexp and continue looping + # Input 1 - regexps | Input 2 - domainQuery + "regex" ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' \ + <(echo "${lists}") <(echo "${domain}") 2>/dev/null;; * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; esac } @@ -97,9 +100,8 @@ scanDatabaseTable() { # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. case "${type}" in - "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; - "retrievetable" ) querystr="SELECT domain FROM vw_${table}";; - * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; + * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; esac # Send prepared query to gravity database @@ -109,13 +111,6 @@ scanDatabaseTable() { return fi - # If we are only retrieving the table - # Just output and return - if [[ "${type}" == "retrievetable" ]]; then - echo "${result[*]}" - return - fi - # Mark domain as having been white-/blacklist matched (global variable) wbMatch=true @@ -138,20 +133,19 @@ scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" # Scan Regex table -regexlist=$(scanDatabaseTable "" "regex" "retrievetable") +mapfile -t regexlist <<< "$(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null)" +# Split results over new line and store in a string +# ready for processing +str_regexlist=$(IFS=$'\n'; echo "${regexlist[*]}") +# If there are regexps in the DB +if [[ -n "${str_regexlist}" ]]; then + # Return any regexps that match the domainQuery + mapfile -t results <<< "$(scanList "${domainQuery}" "${str_regexlist}" "regex")" -if [[ -n "${regexlist}" ]]; then - # Return portion(s) of string that is found in the regex list - mapfile -t results <<< "$(scanList "${domainQuery}" "${regexlist}" "rx")" - - # If a result is found + # If there are matches to the domain query if [[ -n "${results[*]}" ]]; then - # Count the matches - regexCount=${#results[@]} - # Determine plural string - [[ $regexCount -gt 1 ]] && plu="es" # Form output strings - str="${COL_BOLD}${regexCount}${COL_NC} ${matchType}${plu:-} found in ${COL_BOLD}regex${COL_NC} table" + str="${matchType^} found in ${COL_BOLD}regex list${COL_NC}" result="${COL_BOLD}$(IFS=$'\n'; echo "${results[*]}")${COL_NC}" if [[ -z "${blockpage}" ]]; then @@ -160,7 +154,7 @@ if [[ -n "${regexlist}" ]]; then fi case "${blockpage}" in - true ) echo "π ${regexlist##*/}"; exit 0;; + true ) echo "π regex list"; exit 0;; * ) awk '{print " "$0}' <<< "${result}";; esac fi From cf21efa10350cfbb257d0428c6be08594e8139ae Mon Sep 17 00:00:00 2001 From: MMotti Date: Wed, 5 Jun 2019 14:36:43 +0100 Subject: [PATCH 114/207] Minor grammar changes Signed-off-by: MMotti --- advanced/Scripts/query.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 5b8baa63..9134dc0f 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -145,7 +145,7 @@ if [[ -n "${str_regexlist}" ]]; then # If there are matches to the domain query if [[ -n "${results[*]}" ]]; then # Form output strings - str="${matchType^} found in ${COL_BOLD}regex list${COL_NC}" + str="${matchType^} found in ${COL_BOLD}Regex list${COL_NC}" result="${COL_BOLD}$(IFS=$'\n'; echo "${results[*]}")${COL_NC}" if [[ -z "${blockpage}" ]]; then @@ -154,7 +154,7 @@ if [[ -n "${str_regexlist}" ]]; then fi case "${blockpage}" in - true ) echo "π regex list"; exit 0;; + true ) echo "π Regex list"; exit 0;; * ) awk '{print " "$0}' <<< "${result}";; esac fi From 785f1fedd9ade421aa923b997fe873dff171f942 Mon Sep 17 00:00:00 2001 From: MMotti Date: Mon, 10 Jun 2019 17:48:52 +0100 Subject: [PATCH 115/207] Tidy regexp queries Signed-off-by: MMotti --- advanced/Scripts/query.sh | 53 +++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 9134dc0f..e418eecf 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -133,31 +133,36 @@ scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" # Scan Regex table -mapfile -t regexlist <<< "$(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null)" -# Split results over new line and store in a string -# ready for processing -str_regexlist=$(IFS=$'\n'; echo "${regexlist[*]}") -# If there are regexps in the DB -if [[ -n "${str_regexlist}" ]]; then - # Return any regexps that match the domainQuery - mapfile -t results <<< "$(scanList "${domainQuery}" "${str_regexlist}" "regex")" +mapfile -t regexlist < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null) - # If there are matches to the domain query - if [[ -n "${results[*]}" ]]; then - # Form output strings - str="${matchType^} found in ${COL_BOLD}Regex list${COL_NC}" - result="${COL_BOLD}$(IFS=$'\n'; echo "${results[*]}")${COL_NC}" - - if [[ -z "${blockpage}" ]]; then - wcMatch=true - echo " $str" - fi - - case "${blockpage}" in - true ) echo "π Regex list"; exit 0;; - * ) awk '{print " "$0}' <<< "${result}";; - esac - fi +# If we have regexps to process +if [[ "${#regexlist[@]}" -ne 0 ]]; then + # Split regexps over a new line + str_regexlist=$(printf '%s\n' "${regexlist[@]}") + # Check domainQuery against regexps + mapfile -t regexMatches < <(scanList "${domainQuery}" "${str_regexlist}" "regex") + # If there were regex matches + if [[ "${#regexMatches[@]}" -ne 0 ]]; then + # Split matching regexps over a new line + str_regexMatches=$(printf '%s\n' "${regexMatches[@]}") + # Form a "matched" message + str_message="${matchType^} found in ${COL_BOLD}Regex list${COL_NC}" + # Form a "results" message + str_result="${COL_BOLD}${str_regexMatches}${COL_NC}" + + if [[ -z "${blockpage}" ]]; then + # Set the wildcard match flag + wcMatch=true + # Echo the "matched" message, indented by one space + echo " ${str_message}" + # Echo the "results" message, each line indented by three spaces + echo "${str_result}" | sed 's/^/ /' + else + echo "π Regex list" + exit 0 + fi + + fi fi # Get version sorted *.domains filenames (without dir path) From 10fbed50f3e3213a5a1e836cecd144cc59562107 Mon Sep 17 00:00:00 2001 From: MMotti Date: Mon, 10 Jun 2019 18:10:25 +0100 Subject: [PATCH 116/207] Shellcheck Suggesting to use parameter expansion but we need to be able to identify the start of each line Signed-off-by: MMotti --- advanced/Scripts/query.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index e418eecf..93d8baee 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -156,6 +156,7 @@ if [[ "${#regexlist[@]}" -ne 0 ]]; then # Echo the "matched" message, indented by one space echo " ${str_message}" # Echo the "results" message, each line indented by three spaces + # shellcheck disable=SC2001 echo "${str_result}" | sed 's/^/ /' else echo "π Regex list" From a9d0690f4dff7b081f22146cad1440eff04082a1 Mon Sep 17 00:00:00 2001 From: MMotti Date: Mon, 10 Jun 2019 18:50:52 +0100 Subject: [PATCH 117/207] Change the case of some variables Just for consistency Signed-off-by: MMotti --- advanced/Scripts/query.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 93d8baee..aa23ab41 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -133,14 +133,14 @@ scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" # Scan Regex table -mapfile -t regexlist < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null) +mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null) # If we have regexps to process -if [[ "${#regexlist[@]}" -ne 0 ]]; then +if [[ "${#regexList[@]}" -ne 0 ]]; then # Split regexps over a new line - str_regexlist=$(printf '%s\n' "${regexlist[@]}") + str_regexList=$(printf '%s\n' "${regexList[@]}") # Check domainQuery against regexps - mapfile -t regexMatches < <(scanList "${domainQuery}" "${str_regexlist}" "regex") + mapfile -t regexMatches < <(scanList "${domainQuery}" "${str_regexList}" "regex") # If there were regex matches if [[ "${#regexMatches[@]}" -ne 0 ]]; then # Split matching regexps over a new line From bcf03647efd8b1685bf077bfab968c327381de2a Mon Sep 17 00:00:00 2001 From: MMotti Date: Wed, 12 Jun 2019 16:02:59 +0100 Subject: [PATCH 118/207] Add comment and remove blankspace Signed-off-by: MMotti --- advanced/Scripts/query.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index aa23ab41..42ea4395 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -149,7 +149,7 @@ if [[ "${#regexList[@]}" -ne 0 ]]; then str_message="${matchType^} found in ${COL_BOLD}Regex list${COL_NC}" # Form a "results" message str_result="${COL_BOLD}${str_regexMatches}${COL_NC}" - + # If we are displaying more than just the source of the block if [[ -z "${blockpage}" ]]; then # Set the wildcard match flag wcMatch=true @@ -162,7 +162,6 @@ if [[ "${#regexList[@]}" -ne 0 ]]; then echo "π Regex list" exit 0 fi - fi fi From c8987e20c239a59653ae00f01348722440e7b564 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Wed, 12 Jun 2019 19:13:29 -0700 Subject: [PATCH 119/207] Use the filtered IPv6 OpenDNS servers The ones we were using previously were not filtered. See https://support.opendns.com/hc/en-us/articles/227986667-Does-OpenDNS-Support-IPv6- Signed-off-by: Mcat12 --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 29e565ed..1e87b943 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -31,7 +31,7 @@ set -e # List of supported DNS servers DNS_SERVERS=$(cat << EOM Google (ECS);8.8.8.8;8.8.4.4;2001:4860:4860:0:0:0:0:8888;2001:4860:4860:0:0:0:0:8844 -OpenDNS (ECS);208.67.222.222;208.67.220.220;2620:0:ccc::2;2620:0:ccd::2 +OpenDNS (ECS);208.67.222.222;208.67.220.220;2620:119:35::35;2620:119:53::53 Level3;4.2.2.1;4.2.2.2;; Comodo;8.26.56.26;8.20.247.20;; DNS.WATCH;84.200.69.80;84.200.70.40;2001:1608:10:25:0:0:1c04:b12f;2001:1608:10:25:0:0:9249:d69b From 4947350ca5d946a66fdcb7952b4ef200d833e779 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 15 Jun 2019 09:06:10 +0200 Subject: [PATCH 120/207] Add arpflush to help and bash autocompletion Signed-off-by: DL6ER --- advanced/bash-completion/pihole | 2 +- pihole | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 7ba0dad8..cea36060 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -7,7 +7,7 @@ _pihole() { case "${prev}" in "pihole") - opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist" + opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; "whitelist"|"blacklist"|"wildcard"|"regex") diff --git a/pihole b/pihole index d837f873..9fa65a8f 100755 --- a/pihole +++ b/pihole @@ -408,7 +408,8 @@ Options: Add '-h' for more info on disable usage restartdns Restart Pi-hole subsystems checkout Switch Pi-hole subsystems to a different Github branch - Add '-h' for more info on checkout usage"; + Add '-h' for more info on checkout usage + arpflush Flush information stored in Pi-hole's network tables"; exit 0 } From 435a5fb3adb8e38f38772f89260da9cf40441ba0 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 15 Jun 2019 09:12:44 +0200 Subject: [PATCH 121/207] Add pihole arpflush to man page. Signed-off-by: DL6ER --- manpages/pihole.8 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/manpages/pihole.8 b/manpages/pihole.8 index bd7d0933..7a750f4c 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -351,6 +351,12 @@ Switching Pi-hole subsystem branches .br Switch to core development branch .br + +\fBpihole arpflush\fR +.br + Flush information stored in Pi-hole's network tables +.br + .SH "SEE ALSO" \fBlighttpd\fR(8), \fBpihole-FTL\fR(8) From 6996ffa451d4b4911d7394653c840cd738c2a6d6 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 15 Jun 2019 09:15:12 +0200 Subject: [PATCH 122/207] Also flush network_addresses table Signed-off-by: DL6ER --- advanced/Scripts/piholeARPTable.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index 10bee0df..aa45f9ad 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -43,6 +43,16 @@ flushARP(){ return 1 fi + # Truncate network_addresses table in pihole-FTL.db + # This needs to be done before we can truncate the network table due to + # foreign key contraints + if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then + echo -e "${OVER} ${CROSS} Failed to truncate network_addresses table" + echo " Database location: ${DBFILE}" + echo " Output: ${output}" + return 1 + fi + # Truncate network table in pihole-FTL.db if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then echo -e "${OVER} ${CROSS} Failed to truncate network table" From c53ee4202b4baf58283b965d3cb2d617379bf8a4 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 16 Jun 2019 16:50:51 +0200 Subject: [PATCH 123/207] Add filtering by domain groups to gravity database. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 31 +++++++++++++++++++++++++------ 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 372a4a29..ee0caa8e 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,3 +1,13 @@ +PRAGMA FOREIGN_KEYS=ON; + +CREATE TABLE domain_groups +( + "id" INTEGER PRIMARY KEY AUTOINCREMENT, + "enabled" BOOLEAN NOT NULL DEFAULT 1, + "description" TEXT +); +INSERT INTO domain_groups ("id","description") VALUES (0,'Standard group'); + CREATE TABLE whitelist ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -5,7 +15,9 @@ CREATE TABLE whitelist enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + group_id INTEGER DEFAULT 0, + comment TEXT, + FOREIGN KEY (group_id) REFERENCES domain_groups(id) ); CREATE TABLE blacklist ( @@ -14,7 +26,9 @@ CREATE TABLE blacklist enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + group_id INTEGER DEFAULT 0, + comment TEXT, + FOREIGN KEY (group_id) REFERENCES domain_groups(id) ); CREATE TABLE regex ( @@ -23,7 +37,9 @@ CREATE TABLE regex enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + group_id INTEGER DEFAULT 0, + comment TEXT, + FOREIGN KEY (group_id) REFERENCES domain_groups(id) ); CREATE TABLE adlists ( @@ -52,7 +68,8 @@ CREATE VIEW vw_gravity AS SELECT a.domain CREATE VIEW vw_whitelist AS SELECT a.domain FROM whitelist a - WHERE a.enabled == 1 + INNER JOIN domain_groups b ON b.id = a.group_id + WHERE a.enabled = 1 AND b.enabled = 1 ORDER BY a.id; CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist @@ -62,7 +79,8 @@ CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist CREATE VIEW vw_blacklist AS SELECT a.domain FROM blacklist a - WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist + INNER JOIN domain_groups b ON b.id = a.group_id + WHERE a.enabled = 1 AND a.domain NOT IN vw_whitelist AND b.enabled = 1 ORDER BY a.id; CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist @@ -72,7 +90,8 @@ CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist CREATE VIEW vw_regex AS SELECT a.domain FROM regex a - WHERE a.enabled == 1 + INNER JOIN domain_groups b ON b.id = a.group_id + WHERE a.enabled = 1 AND b.enabled = 1 ORDER BY a.id; CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex From cbc210b0145415a6aedfc28c6ae266fd0a85c3ec Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 16 Jun 2019 17:05:33 +0200 Subject: [PATCH 124/207] Add adlist_groups and mark group_id columns as NOT NULL. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 22 +++++++++++++++++----- gravity.sh | 2 +- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index ee0caa8e..e22cc44b 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -15,7 +15,7 @@ CREATE TABLE whitelist enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - group_id INTEGER DEFAULT 0, + group_id INTEGER NOT NULL DEFAULT 0, comment TEXT, FOREIGN KEY (group_id) REFERENCES domain_groups(id) ); @@ -26,7 +26,7 @@ CREATE TABLE blacklist enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - group_id INTEGER DEFAULT 0, + group_id INTEGER NOT NULL DEFAULT 0, comment TEXT, FOREIGN KEY (group_id) REFERENCES domain_groups(id) ); @@ -37,10 +37,19 @@ CREATE TABLE regex enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - group_id INTEGER DEFAULT 0, + group_id INTEGER NOT NULL DEFAULT 0, comment TEXT, FOREIGN KEY (group_id) REFERENCES domain_groups(id) ); + +CREATE TABLE adlist_groups +( + "id" INTEGER PRIMARY KEY AUTOINCREMENT, + "enabled" BOOLEAN NOT NULL DEFAULT 1, + "description" TEXT +); +INSERT INTO adlist_groups ("id","description") VALUES (0,'Standard group'); + CREATE TABLE adlists ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -48,7 +57,9 @@ CREATE TABLE adlists enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + group_id INTEGER NOT NULL DEFAULT 0, + comment TEXT, + FOREIGN KEY (group_id) REFERENCES adlist_groups(id) ); CREATE TABLE gravity ( @@ -101,7 +112,8 @@ CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex CREATE VIEW vw_adlists AS SELECT a.address FROM adlists a - WHERE a.enabled == 1 + INNER JOIN adlist_groups b ON b.id = a.group_id + WHERE a.enabled = 1 AND b.enabled = 1 ORDER BY a.id; CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists diff --git a/gravity.sh b/gravity.sh index 1fe2b4e3..ce7aa6ba 100755 --- a/gravity.sh +++ b/gravity.sh @@ -120,7 +120,7 @@ database_table_from_file() { do # Only add non-empty lines if [[ ! -z "${domain}" ]]; then - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},0,\"Migrated from ${source}\"" >> "${tmpFile}" rowid+=1 fi done From 01850c2128f2f9889db352028d1c296247153ca5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 16 Jun 2019 17:15:36 +0200 Subject: [PATCH 125/207] Automatically recreate standard group after deletion to avoid violating foreign key constraints even if foreign key enforcing is disabled (SQLite's default behavior) Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index e22cc44b..5fb9a5eb 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -7,6 +7,10 @@ CREATE TABLE domain_groups "description" TEXT ); INSERT INTO domain_groups ("id","description") VALUES (0,'Standard group'); +CREATE TRIGGER domain_groups_standard_group AFTER DELETE ON domain_groups WHEN OLD.id = 0 + BEGIN + INSERT INTO domain_groups ("id","description") VALUES (0,'Standard group'); + END; CREATE TABLE whitelist ( @@ -49,6 +53,10 @@ CREATE TABLE adlist_groups "description" TEXT ); INSERT INTO adlist_groups ("id","description") VALUES (0,'Standard group'); +CREATE TRIGGER adlist_groups_standard_group AFTER DELETE ON adlist_groups WHEN OLD.id = 0 + BEGIN + INSERT INTO adlist_groups ("id","description") VALUES (0,'Standard group'); + END; CREATE TABLE adlists ( From 0774c4e5cac3acd7955ef9a53d19f0a4ed69c13b Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 16 Jun 2019 18:52:23 +0200 Subject: [PATCH 126/207] vw_gravity should actually filter using vw_whitelist. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 5fb9a5eb..f4f9cf4c 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -83,7 +83,7 @@ INSERT INTO info VALUES("version","1"); CREATE VIEW vw_gravity AS SELECT a.domain FROM gravity a - WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); + WHERE a.domain NOT IN (SELECT domain from vw_whitelist); CREATE VIEW vw_whitelist AS SELECT a.domain FROM whitelist a From d67122dffc82456040960abb62c02b8a7103ab22 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 16 Jun 2019 11:13:23 -0700 Subject: [PATCH 127/207] Pin the Fedora test docker container to 29 The tests are currently failing due to a change in Fedora 30, which we are not currently supporting. Signed-off-by: Mcat12 --- test/fedora.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fedora.Dockerfile b/test/fedora.Dockerfile index c4834388..8c27bbcc 100644 --- a/test/fedora.Dockerfile +++ b/test/fedora.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:latest +FROM fedora:29 ENV GITDIR /etc/.pihole ENV SCRIPTDIR /opt/pihole From b6700924b23a580a5286d8af343e205ae2d807d5 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 17 Jun 2019 16:13:31 -0700 Subject: [PATCH 128/207] Fix uninstall causing 403 errors and not removing packages The 403 lighttpd errors were caused by removing the lighttpd config directory and not removing lighttpd itself. This caused a subsequent Pi-hole reinstall to not have all of the required lighttpd config files. The error while removing packages was caused by combining arguments into a string instead of listing each argument. Signed-off-by: Mcat12 --- automated install/uninstall.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 732fc246..8e5c5455 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -80,7 +80,7 @@ removeAndPurge() { case ${yn} in [Yy]* ) echo -ne " ${INFO} Removing ${i}..."; - ${SUDO} "${PKG_REMOVE} ${i}" &> /dev/null; + ${SUDO} ${PKG_REMOVE} "${i}" &> /dev/null; echo -e "${OVER} ${INFO} Removed ${i}"; break;; [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; @@ -132,12 +132,15 @@ removeNoPurge() { fi if package_check lighttpd > /dev/null; then - ${SUDO} rm -rf /etc/lighttpd/ &> /dev/null - echo -e " ${TICK} Removed lighttpd" - else - if [ -f /etc/lighttpd/lighttpd.conf.orig ]; then + if [[ -f /etc/lighttpd/lighttpd.conf.orig ]]; then ${SUDO} mv /etc/lighttpd/lighttpd.conf.orig /etc/lighttpd/lighttpd.conf fi + + if [[ -f /etc/lighttpd/external.conf ]]; then + ${SUDO} rm /etc/lighttpd/external.conf + fi + + echo -e " ${TICK} Removed lighttpd configs" fi ${SUDO} rm -f /etc/dnsmasq.d/adList.conf &> /dev/null From 951732fc1b07823e8f7c6b4cbfd28f753f1a8e50 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 17 Jun 2019 16:28:55 -0700 Subject: [PATCH 129/207] Use an array for PKG_REMOVE Fixes shellcheck warning. Signed-off-by: Mcat12 --- automated install/uninstall.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 8e5c5455..1f58b887 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -55,13 +55,13 @@ fi # Compatability if [ -x "$(command -v apt-get)" ]; then # Debian Family - PKG_REMOVE="${PKG_MANAGER} -y remove --purge" + PKG_REMOVE=("${PKG_MANAGER}" -y remove --purge) package_check() { dpkg-query -W -f='${Status}' "$1" 2>/dev/null | grep -c "ok installed" } elif [ -x "$(command -v rpm)" ]; then # Fedora Family - PKG_REMOVE="${PKG_MANAGER} remove -y" + PKG_REMOVE=("${PKG_MANAGER}" remove -y) package_check() { rpm -qa | grep "^$1-" > /dev/null } @@ -80,7 +80,7 @@ removeAndPurge() { case ${yn} in [Yy]* ) echo -ne " ${INFO} Removing ${i}..."; - ${SUDO} ${PKG_REMOVE} "${i}" &> /dev/null; + ${SUDO} "${PKG_REMOVE}" "${i}" &> /dev/null; echo -e "${OVER} ${INFO} Removed ${i}"; break;; [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; From 974bba4a45427e628165e6e229984b810dcec2de Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 17 Jun 2019 16:30:26 -0700 Subject: [PATCH 130/207] Fix PKG_REMOVE array usage Signed-off-by: Mcat12 --- automated install/uninstall.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 1f58b887..2d6837b4 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -80,7 +80,7 @@ removeAndPurge() { case ${yn} in [Yy]* ) echo -ne " ${INFO} Removing ${i}..."; - ${SUDO} "${PKG_REMOVE}" "${i}" &> /dev/null; + ${SUDO} "${PKG_REMOVE[@]}" "${i}" &> /dev/null; echo -e "${OVER} ${INFO} Removed ${i}"; break;; [Nn]* ) echo -e " ${INFO} Skipped ${i}"; break;; From d5e8f1a781bd21caec692b40949fb3af436c5c07 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 26 Jun 2019 18:20:36 +0200 Subject: [PATCH 131/207] Add 10 seconds timeout to the sqlite3 command that writes tables to the gravity database. This prevents gravity from failing when the database file is locked for a short time. Signed-off-by: DL6ER --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index 1fe2b4e3..1c142ffd 100755 --- a/gravity.sh +++ b/gravity.sh @@ -129,7 +129,7 @@ database_table_from_file() { # Store domains in database table specified by ${table} # Use printf as .mode and .import need to be on separate lines # see https://unix.stackexchange.com/a/445615/83260 - output=$( { printf ".mode csv\\n.import \"%s\" %s\\n" "${inputfile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + output=$( { printf ".timeout 10000\\n.mode csv\\n.import \"%s\" %s\\n" "${inputfile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) status="$?" if [[ "${status}" -ne 0 ]]; then From 91a2d052a7975d543e35d2d38aa65293e3b11e46 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 28 Jun 2019 20:49:56 -0700 Subject: [PATCH 132/207] Fix pihole -up showing FTL update when network is down Fixes #1877 Signed-off-by: Mcat12 --- automated install/basic-install.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1e87b943..baeaaa9d 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2402,6 +2402,11 @@ FTLcheckUpdate() { local FTLlatesttag FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep 'Location' | awk -F '/' '{print $NF}' | tr -d '\r\n') + if [[ $? != 0 ]]; then + # There was an issue while retrieving the latest version + return 3 + fi + if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then return 0 else From 37e7cd5211516e87fe84e5b908b93ae9aee6e2d5 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 28 Jun 2019 20:57:05 -0700 Subject: [PATCH 133/207] Fix ShellCheck issue by refactoring a bit Signed-off-by: Mcat12 --- automated install/basic-install.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index baeaaa9d..380b424f 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2399,14 +2399,16 @@ FTLcheckUpdate() { if [[ ${ftlLoc} ]]; then local FTLversion FTLversion=$(/usr/bin/pihole-FTL tag) + local FTLreleaseData local FTLlatesttag - FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep 'Location' | awk -F '/' '{print $NF}' | tr -d '\r\n') - if [[ $? != 0 ]]; then + if ! FTLreleaseData=$(curl -sI https://github.com/pi-hole/FTL/releases/latest); then # There was an issue while retrieving the latest version return 3 fi + FTLlatesttag=$(grep 'Location' < "${FTLreleaseData}" | awk -F '/' '{print $NF}' | tr -d '\r\n') + if [[ "${FTLversion}" != "${FTLlatesttag}" ]]; then return 0 else From acee97916e3d9b22a8afb81230303c9a90a745a3 Mon Sep 17 00:00:00 2001 From: Andreas Kurth Date: Sun, 30 Jun 2019 08:02:51 +0200 Subject: [PATCH 134/207] Fix pihole manpage to match code. The dry-run argument to pihole -up is "--check-only", not "--checkonly". Signed-off-by: Andreas Kurth --- manpages/pihole.8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manpages/pihole.8 b/manpages/pihole.8 index bd7d0933..b3008563 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -35,7 +35,7 @@ pihole -g\fR .br \fBpihole\fR \fB-l\fR (\fBon|off|off noflush\fR) .br -\fBpihole -up \fR[--checkonly] +\fBpihole -up \fR[--check-only] .br \fBpihole -v\fR [-p|-a|-f] [-c|-l|-hash] .br From 87c115fc86bd0365e82d5b46f911c0c52b228463 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Jun 2019 18:58:47 +0200 Subject: [PATCH 135/207] Change implementation to use linking tables (domain_groups, adlists_groups). This allows adding domains to multiple lists conveniently. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 121 +++++++++++++++++------------- gravity.sh | 2 +- 2 files changed, 70 insertions(+), 53 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index f4f9cf4c..77de31aa 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -2,15 +2,10 @@ PRAGMA FOREIGN_KEYS=ON; CREATE TABLE domain_groups ( - "id" INTEGER PRIMARY KEY AUTOINCREMENT, - "enabled" BOOLEAN NOT NULL DEFAULT 1, - "description" TEXT + id INTEGER PRIMARY KEY AUTOINCREMENT, + enabled BOOLEAN NOT NULL DEFAULT 1, + description TEXT ); -INSERT INTO domain_groups ("id","description") VALUES (0,'Standard group'); -CREATE TRIGGER domain_groups_standard_group AFTER DELETE ON domain_groups WHEN OLD.id = 0 - BEGIN - INSERT INTO domain_groups ("id","description") VALUES (0,'Standard group'); - END; CREATE TABLE whitelist ( @@ -19,10 +14,16 @@ CREATE TABLE whitelist enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - group_id INTEGER NOT NULL DEFAULT 0, - comment TEXT, - FOREIGN KEY (group_id) REFERENCES domain_groups(id) + comment TEXT ); + +CREATE TABLE whitelist_by_group +( + whitelist_id INTEGER NOT NULL REFERENCES whitelist (id), + group_id INTEGER NOT NULL REFERENCES domain_groups (id), + PRIMARY KEY (whitelist_id, group_id) +); + CREATE TABLE blacklist ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -30,10 +31,16 @@ CREATE TABLE blacklist enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - group_id INTEGER NOT NULL DEFAULT 0, - comment TEXT, - FOREIGN KEY (group_id) REFERENCES domain_groups(id) + comment TEXT ); + +CREATE TABLE blacklist_by_group +( + blacklist_id INTEGER NOT NULL REFERENCES blacklist (id), + group_id INTEGER NOT NULL REFERENCES domain_groups (id), + PRIMARY KEY (blacklist_id, group_id) +); + CREATE TABLE regex ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -41,22 +48,22 @@ CREATE TABLE regex enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - group_id INTEGER NOT NULL DEFAULT 0, - comment TEXT, - FOREIGN KEY (group_id) REFERENCES domain_groups(id) + comment TEXT ); -CREATE TABLE adlist_groups +CREATE TABLE regex_by_group ( - "id" INTEGER PRIMARY KEY AUTOINCREMENT, - "enabled" BOOLEAN NOT NULL DEFAULT 1, - "description" TEXT + regex_id INTEGER NOT NULL REFERENCES regex (id), + group_id INTEGER NOT NULL REFERENCES domain_groups (id), + PRIMARY KEY (regex_id, group_id) +); + +CREATE TABLE adlists_groups +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + enabled BOOLEAN NOT NULL DEFAULT 1, + description TEXT ); -INSERT INTO adlist_groups ("id","description") VALUES (0,'Standard group'); -CREATE TRIGGER adlist_groups_standard_group AFTER DELETE ON adlist_groups WHEN OLD.id = 0 - BEGIN - INSERT INTO adlist_groups ("id","description") VALUES (0,'Standard group'); - END; CREATE TABLE adlists ( @@ -65,10 +72,16 @@ CREATE TABLE adlists enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - group_id INTEGER NOT NULL DEFAULT 0, - comment TEXT, - FOREIGN KEY (group_id) REFERENCES adlist_groups(id) + comment TEXT ); + +CREATE TABLE adlists_by_group +( + adlists_id INTEGER NOT NULL REFERENCES adlists (id), + group_id INTEGER NOT NULL REFERENCES adlists_groups (id), + PRIMARY KEY (adlists_id, group_id) +); + CREATE TABLE gravity ( domain TEXT PRIMARY KEY @@ -81,48 +94,52 @@ CREATE TABLE info INSERT INTO info VALUES("version","1"); -CREATE VIEW vw_gravity AS SELECT a.domain - FROM gravity a - WHERE a.domain NOT IN (SELECT domain from vw_whitelist); +CREATE VIEW vw_gravity AS SELECT domain + FROM gravity + WHERE domain NOT IN (SELECT domain from vw_whitelist); -CREATE VIEW vw_whitelist AS SELECT a.domain - FROM whitelist a - INNER JOIN domain_groups b ON b.id = a.group_id - WHERE a.enabled = 1 AND b.enabled = 1 - ORDER BY a.id; +CREATE VIEW vw_whitelist AS SELECT domain + FROM whitelist + LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id + LEFT JOIN domain_groups ON domain_groups.id = whitelist_by_group.group_id + WHERE whitelist.enabled = 1 AND domain_groups.enabled IS NULL OR domain_groups.enabled == 1 + ORDER BY whitelist.id; CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist BEGIN UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_blacklist AS SELECT a.domain - FROM blacklist a - INNER JOIN domain_groups b ON b.id = a.group_id - WHERE a.enabled = 1 AND a.domain NOT IN vw_whitelist AND b.enabled = 1 - ORDER BY a.id; +CREATE VIEW vw_blacklist AS SELECT domain + FROM blacklist + LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id + LEFT JOIN domain_groups ON domain_groups.id = blacklist_by_group.group_id + WHERE blacklist.enabled = 1 AND domain_groups.enabled IS NULL OR domain_groups.enabled == 1 + ORDER BY blacklist.id; CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist BEGIN UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_regex AS SELECT a.domain - FROM regex a - INNER JOIN domain_groups b ON b.id = a.group_id - WHERE a.enabled = 1 AND b.enabled = 1 - ORDER BY a.id; +CREATE VIEW vw_regex AS SELECT domain + FROM regex + LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id + LEFT JOIN domain_groups ON domain_groups.id = regex_by_group.group_id + WHERE regex.enabled = 1 AND domain_groups.enabled IS NULL OR domain_groups.enabled == 1 + ORDER BY regex.id; CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex BEGIN UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_adlists AS SELECT a.address - FROM adlists a - INNER JOIN adlist_groups b ON b.id = a.group_id - WHERE a.enabled = 1 AND b.enabled = 1 - ORDER BY a.id; +CREATE VIEW vw_adlists AS SELECT address + FROM adlists + LEFT JOIN adlists_by_group ON adlists_by_group.adlists_id = adlists.id + LEFT JOIN adlists_groups ON adlists_groups.id = adlists_by_group.group_id + WHERE adlists.enabled = 1 AND adlists_groups.enabled IS NULL OR adlists_groups.enabled == 1 + ORDER BY adlists.id; CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists BEGIN diff --git a/gravity.sh b/gravity.sh index ce7aa6ba..1fe2b4e3 100755 --- a/gravity.sh +++ b/gravity.sh @@ -120,7 +120,7 @@ database_table_from_file() { do # Only add non-empty lines if [[ ! -z "${domain}" ]]; then - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},0,\"Migrated from ${source}\"" >> "${tmpFile}" + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" rowid+=1 fi done From 94d83dbb2d410c0303ec09ab7e5365cb4772cb10 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Jun 2019 19:33:02 +0200 Subject: [PATCH 136/207] Unify the two group tables into a single "groups" table. This allows managing adlists and domains together. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 33 ++++++++++++------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 77de31aa..dfb486a8 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,6 +1,6 @@ PRAGMA FOREIGN_KEYS=ON; -CREATE TABLE domain_groups +CREATE TABLE groups ( id INTEGER PRIMARY KEY AUTOINCREMENT, enabled BOOLEAN NOT NULL DEFAULT 1, @@ -20,7 +20,7 @@ CREATE TABLE whitelist CREATE TABLE whitelist_by_group ( whitelist_id INTEGER NOT NULL REFERENCES whitelist (id), - group_id INTEGER NOT NULL REFERENCES domain_groups (id), + group_id INTEGER NOT NULL REFERENCES groups (id), PRIMARY KEY (whitelist_id, group_id) ); @@ -37,7 +37,7 @@ CREATE TABLE blacklist CREATE TABLE blacklist_by_group ( blacklist_id INTEGER NOT NULL REFERENCES blacklist (id), - group_id INTEGER NOT NULL REFERENCES domain_groups (id), + group_id INTEGER NOT NULL REFERENCES groups (id), PRIMARY KEY (blacklist_id, group_id) ); @@ -54,17 +54,10 @@ CREATE TABLE regex CREATE TABLE regex_by_group ( regex_id INTEGER NOT NULL REFERENCES regex (id), - group_id INTEGER NOT NULL REFERENCES domain_groups (id), + group_id INTEGER NOT NULL REFERENCES groups (id), PRIMARY KEY (regex_id, group_id) ); -CREATE TABLE adlists_groups -( - id INTEGER PRIMARY KEY AUTOINCREMENT, - enabled BOOLEAN NOT NULL DEFAULT 1, - description TEXT -); - CREATE TABLE adlists ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -78,7 +71,7 @@ CREATE TABLE adlists CREATE TABLE adlists_by_group ( adlists_id INTEGER NOT NULL REFERENCES adlists (id), - group_id INTEGER NOT NULL REFERENCES adlists_groups (id), + group_id INTEGER NOT NULL REFERENCES groups (id), PRIMARY KEY (adlists_id, group_id) ); @@ -101,8 +94,8 @@ CREATE VIEW vw_gravity AS SELECT domain CREATE VIEW vw_whitelist AS SELECT domain FROM whitelist LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id - LEFT JOIN domain_groups ON domain_groups.id = whitelist_by_group.group_id - WHERE whitelist.enabled = 1 AND domain_groups.enabled IS NULL OR domain_groups.enabled == 1 + LEFT JOIN groups ON groups.id = whitelist_by_group.group_id + WHERE whitelist.enabled = 1 AND groups.enabled IS NULL OR groups.enabled == 1 ORDER BY whitelist.id; CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist @@ -113,8 +106,8 @@ CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist CREATE VIEW vw_blacklist AS SELECT domain FROM blacklist LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id - LEFT JOIN domain_groups ON domain_groups.id = blacklist_by_group.group_id - WHERE blacklist.enabled = 1 AND domain_groups.enabled IS NULL OR domain_groups.enabled == 1 + LEFT JOIN groups ON groups.id = blacklist_by_group.group_id + WHERE blacklist.enabled = 1 AND groups.enabled IS NULL OR groups.enabled == 1 ORDER BY blacklist.id; CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist @@ -125,8 +118,8 @@ CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist CREATE VIEW vw_regex AS SELECT domain FROM regex LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id - LEFT JOIN domain_groups ON domain_groups.id = regex_by_group.group_id - WHERE regex.enabled = 1 AND domain_groups.enabled IS NULL OR domain_groups.enabled == 1 + LEFT JOIN groups ON groups.id = regex_by_group.group_id + WHERE regex.enabled = 1 AND groups.enabled IS NULL OR groups.enabled == 1 ORDER BY regex.id; CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex @@ -137,8 +130,8 @@ CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex CREATE VIEW vw_adlists AS SELECT address FROM adlists LEFT JOIN adlists_by_group ON adlists_by_group.adlists_id = adlists.id - LEFT JOIN adlists_groups ON adlists_groups.id = adlists_by_group.group_id - WHERE adlists.enabled = 1 AND adlists_groups.enabled IS NULL OR adlists_groups.enabled == 1 + LEFT JOIN groups ON groups.id = adlists_by_group.group_id + WHERE adlists.enabled = 1 AND groups.enabled IS NULL OR groups.enabled == 1 ORDER BY adlists.id; CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists From 6fe637b9ee18e657c2806ba2ba3d1c86c5e8b2bc Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Jun 2019 19:43:18 +0200 Subject: [PATCH 137/207] Rename groups -> group Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index dfb486a8..3c2562b4 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,6 +1,6 @@ PRAGMA FOREIGN_KEYS=ON; -CREATE TABLE groups +CREATE TABLE group ( id INTEGER PRIMARY KEY AUTOINCREMENT, enabled BOOLEAN NOT NULL DEFAULT 1, @@ -20,7 +20,7 @@ CREATE TABLE whitelist CREATE TABLE whitelist_by_group ( whitelist_id INTEGER NOT NULL REFERENCES whitelist (id), - group_id INTEGER NOT NULL REFERENCES groups (id), + group_id INTEGER NOT NULL REFERENCES group (id), PRIMARY KEY (whitelist_id, group_id) ); @@ -37,7 +37,7 @@ CREATE TABLE blacklist CREATE TABLE blacklist_by_group ( blacklist_id INTEGER NOT NULL REFERENCES blacklist (id), - group_id INTEGER NOT NULL REFERENCES groups (id), + group_id INTEGER NOT NULL REFERENCES group (id), PRIMARY KEY (blacklist_id, group_id) ); @@ -54,7 +54,7 @@ CREATE TABLE regex CREATE TABLE regex_by_group ( regex_id INTEGER NOT NULL REFERENCES regex (id), - group_id INTEGER NOT NULL REFERENCES groups (id), + group_id INTEGER NOT NULL REFERENCES group (id), PRIMARY KEY (regex_id, group_id) ); @@ -71,7 +71,7 @@ CREATE TABLE adlists CREATE TABLE adlists_by_group ( adlists_id INTEGER NOT NULL REFERENCES adlists (id), - group_id INTEGER NOT NULL REFERENCES groups (id), + group_id INTEGER NOT NULL REFERENCES group (id), PRIMARY KEY (adlists_id, group_id) ); @@ -94,8 +94,8 @@ CREATE VIEW vw_gravity AS SELECT domain CREATE VIEW vw_whitelist AS SELECT domain FROM whitelist LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id - LEFT JOIN groups ON groups.id = whitelist_by_group.group_id - WHERE whitelist.enabled = 1 AND groups.enabled IS NULL OR groups.enabled == 1 + LEFT JOIN group ON group.id = whitelist_by_group.group_id + WHERE whitelist.enabled = 1 AND group.enabled IS NULL OR group.enabled = 1 ORDER BY whitelist.id; CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist @@ -106,8 +106,8 @@ CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist CREATE VIEW vw_blacklist AS SELECT domain FROM blacklist LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id - LEFT JOIN groups ON groups.id = blacklist_by_group.group_id - WHERE blacklist.enabled = 1 AND groups.enabled IS NULL OR groups.enabled == 1 + LEFT JOIN group ON group.id = blacklist_by_group.group_id + WHERE blacklist.enabled = 1 AND group.enabled IS NULL OR group.enabled = 1 ORDER BY blacklist.id; CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist @@ -118,8 +118,8 @@ CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist CREATE VIEW vw_regex AS SELECT domain FROM regex LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id - LEFT JOIN groups ON groups.id = regex_by_group.group_id - WHERE regex.enabled = 1 AND groups.enabled IS NULL OR groups.enabled == 1 + LEFT JOIN group ON group.id = regex_by_group.group_id + WHERE regex.enabled = 1 AND group.enabled IS NULL OR group.enabled = 1 ORDER BY regex.id; CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex @@ -130,8 +130,8 @@ CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex CREATE VIEW vw_adlists AS SELECT address FROM adlists LEFT JOIN adlists_by_group ON adlists_by_group.adlists_id = adlists.id - LEFT JOIN groups ON groups.id = adlists_by_group.group_id - WHERE adlists.enabled = 1 AND groups.enabled IS NULL OR groups.enabled == 1 + LEFT JOIN group ON group.id = adlists_by_group.group_id + WHERE adlists.enabled = 1 AND group.enabled IS NULL OR group.enabled = 1 ORDER BY adlists.id; CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists From 5b01facd73f52ccb248a65109b45a62e327c597d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Jun 2019 20:03:22 +0200 Subject: [PATCH 138/207] Fix logix on the views: AND ( OR ) Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 3c2562b4..66df7111 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -1,6 +1,6 @@ PRAGMA FOREIGN_KEYS=ON; -CREATE TABLE group +CREATE TABLE "group" ( id INTEGER PRIMARY KEY AUTOINCREMENT, enabled BOOLEAN NOT NULL DEFAULT 1, @@ -20,7 +20,7 @@ CREATE TABLE whitelist CREATE TABLE whitelist_by_group ( whitelist_id INTEGER NOT NULL REFERENCES whitelist (id), - group_id INTEGER NOT NULL REFERENCES group (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), PRIMARY KEY (whitelist_id, group_id) ); @@ -37,7 +37,7 @@ CREATE TABLE blacklist CREATE TABLE blacklist_by_group ( blacklist_id INTEGER NOT NULL REFERENCES blacklist (id), - group_id INTEGER NOT NULL REFERENCES group (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), PRIMARY KEY (blacklist_id, group_id) ); @@ -54,7 +54,7 @@ CREATE TABLE regex CREATE TABLE regex_by_group ( regex_id INTEGER NOT NULL REFERENCES regex (id), - group_id INTEGER NOT NULL REFERENCES group (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), PRIMARY KEY (regex_id, group_id) ); @@ -71,7 +71,7 @@ CREATE TABLE adlists CREATE TABLE adlists_by_group ( adlists_id INTEGER NOT NULL REFERENCES adlists (id), - group_id INTEGER NOT NULL REFERENCES group (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), PRIMARY KEY (adlists_id, group_id) ); @@ -94,8 +94,8 @@ CREATE VIEW vw_gravity AS SELECT domain CREATE VIEW vw_whitelist AS SELECT domain FROM whitelist LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id - LEFT JOIN group ON group.id = whitelist_by_group.group_id - WHERE whitelist.enabled = 1 AND group.enabled IS NULL OR group.enabled = 1 + LEFT JOIN "group" ON "group".id = whitelist_by_group.group_id + WHERE whitelist.enabled = 1 AND ("group".enabled IS NULL OR "group".enabled = 1) ORDER BY whitelist.id; CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist @@ -106,8 +106,8 @@ CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist CREATE VIEW vw_blacklist AS SELECT domain FROM blacklist LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id - LEFT JOIN group ON group.id = blacklist_by_group.group_id - WHERE blacklist.enabled = 1 AND group.enabled IS NULL OR group.enabled = 1 + LEFT JOIN "group" ON "group".id = blacklist_by_group.group_id + WHERE blacklist.enabled = 1 AND ("group".enabled IS NULL OR "group".enabled = 1) ORDER BY blacklist.id; CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist @@ -118,8 +118,8 @@ CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist CREATE VIEW vw_regex AS SELECT domain FROM regex LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id - LEFT JOIN group ON group.id = regex_by_group.group_id - WHERE regex.enabled = 1 AND group.enabled IS NULL OR group.enabled = 1 + LEFT JOIN "group" ON "group".id = regex_by_group.group_id + WHERE regex.enabled = 1 AND ("group".enabled IS NULL OR "group".enabled = 1) ORDER BY regex.id; CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex @@ -130,8 +130,8 @@ CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex CREATE VIEW vw_adlists AS SELECT address FROM adlists LEFT JOIN adlists_by_group ON adlists_by_group.adlists_id = adlists.id - LEFT JOIN group ON group.id = adlists_by_group.group_id - WHERE adlists.enabled = 1 AND group.enabled IS NULL OR group.enabled = 1 + LEFT JOIN "group" ON "group".id = adlists_by_group.group_id + WHERE adlists.enabled = 1 AND ("group".enabled IS NULL OR "group".enabled = 1) ORDER BY adlists.id; CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists From ad97e95f2fbf101011ac053793dc72da15568c99 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Jun 2019 23:06:15 +0200 Subject: [PATCH 139/207] Add required name column in group table. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 66df7111..9be4bbf9 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -4,6 +4,7 @@ CREATE TABLE "group" ( id INTEGER PRIMARY KEY AUTOINCREMENT, enabled BOOLEAN NOT NULL DEFAULT 1, + name TEXT NOT NULL, description TEXT ); From 3a14e8b013a0ebc5e8133d78153920d6326b7fd9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Jun 2019 23:07:03 +0200 Subject: [PATCH 140/207] Check whitelist_by_group.group_id instead of group.enabled against NULL. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 9be4bbf9..a5fd5bef 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -96,7 +96,7 @@ CREATE VIEW vw_whitelist AS SELECT domain FROM whitelist LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id LEFT JOIN "group" ON "group".id = whitelist_by_group.group_id - WHERE whitelist.enabled = 1 AND ("group".enabled IS NULL OR "group".enabled = 1) + WHERE whitelist.enabled = 1 AND (whitelist_by_group.group_id IS NULL OR "group".enabled = 1) ORDER BY whitelist.id; CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist @@ -108,7 +108,7 @@ CREATE VIEW vw_blacklist AS SELECT domain FROM blacklist LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id LEFT JOIN "group" ON "group".id = blacklist_by_group.group_id - WHERE blacklist.enabled = 1 AND ("group".enabled IS NULL OR "group".enabled = 1) + WHERE blacklist.enabled = 1 AND (blacklist_by_group.group_id IS NULL OR "group".enabled = 1) ORDER BY blacklist.id; CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist @@ -120,7 +120,7 @@ CREATE VIEW vw_regex AS SELECT domain FROM regex LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id LEFT JOIN "group" ON "group".id = regex_by_group.group_id - WHERE regex.enabled = 1 AND ("group".enabled IS NULL OR "group".enabled = 1) + WHERE regex.enabled = 1 AND (regex_by_group.group_id IS NULL OR "group".enabled = 1) ORDER BY regex.id; CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex @@ -132,7 +132,7 @@ CREATE VIEW vw_adlists AS SELECT address FROM adlists LEFT JOIN adlists_by_group ON adlists_by_group.adlists_id = adlists.id LEFT JOIN "group" ON "group".id = adlists_by_group.group_id - WHERE adlists.enabled = 1 AND ("group".enabled IS NULL OR "group".enabled = 1) + WHERE adlists.enabled = 1 AND (adlists_by_group.group_id IS NULL OR "group".enabled = 1) ORDER BY adlists.id; CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists From f91421418ad3817023b53585a351a2778b644034 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 30 Jun 2019 23:21:10 +0200 Subject: [PATCH 141/207] Rename adlists -> adlist table. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 24 ++++++++++++------------ gravity.sh | 10 +++++----- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index a5fd5bef..3cbc7b8f 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -59,7 +59,7 @@ CREATE TABLE regex_by_group PRIMARY KEY (regex_id, group_id) ); -CREATE TABLE adlists +CREATE TABLE adlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, address TEXT UNIQUE NOT NULL, @@ -69,11 +69,11 @@ CREATE TABLE adlists comment TEXT ); -CREATE TABLE adlists_by_group +CREATE TABLE adlist_by_group ( - adlists_id INTEGER NOT NULL REFERENCES adlists (id), + adlist_id INTEGER NOT NULL REFERENCES adlist (id), group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (adlists_id, group_id) + PRIMARY KEY (adlist_id, group_id) ); CREATE TABLE gravity @@ -128,15 +128,15 @@ CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_adlists AS SELECT address - FROM adlists - LEFT JOIN adlists_by_group ON adlists_by_group.adlists_id = adlists.id - LEFT JOIN "group" ON "group".id = adlists_by_group.group_id - WHERE adlists.enabled = 1 AND (adlists_by_group.group_id IS NULL OR "group".enabled = 1) - ORDER BY adlists.id; +CREATE VIEW vw_adlist AS SELECT address + FROM adlist + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY adlist.id; -CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists +CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist BEGIN - UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; + UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; END; diff --git a/gravity.sh b/gravity.sh index 1fe2b4e3..d2436002 100755 --- a/gravity.sh +++ b/gravity.sh @@ -111,7 +111,7 @@ database_table_from_file() { # No need to modify the input data for the gravity table inputfile="${source}" else - # Apply format for white-, blacklist, regex, and adlists tables + # Apply format for white-, blacklist, regex, and adlist tables local rowid declare -i rowid rowid=1 @@ -159,9 +159,9 @@ migrate_to_database() { # Migrate list files to new database if [[ -e "${adListFile}" ]]; then - # Store adlists domains in database + # Store adlist domains in database echo -e " ${INFO} Migrating content of ${adListFile} into new database" - database_table_from_file "adlists" "${adListFile}" + database_table_from_file "adlist" "${adListFile}" fi if [[ -e "${blacklistFile}" ]]; then # Store blacklisted domains in database @@ -236,13 +236,13 @@ gravity_CheckDNSResolutionAvailable() { gravity_CheckDNSResolutionAvailable } -# Retrieve blocklist URLs and parse domains from adlists.list +# Retrieve blocklist URLs and parse domains from adlist.list gravity_GetBlocklistUrls() { echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..." # Retrieve source URLs from gravity database # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true) - mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" + mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( From 7b2ac25a52b71fae5aac79742ad41c263cb402f0 Mon Sep 17 00:00:00 2001 From: Jeroen Baert <3607063+Forceflow@users.noreply.github.com> Date: Mon, 1 Jul 2019 03:42:02 +0200 Subject: [PATCH 142/207] Fix for 404 error when browsing to pi.hole (without /admin) Signed-off-by: Jeroen Baert <3607063+Forceflow@users.noreply.github.com> --- advanced/lighttpd.conf.debian | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian index 47bdbee0..2215bbdb 100644 --- a/advanced/lighttpd.conf.debian +++ b/advanced/lighttpd.conf.debian @@ -27,7 +27,7 @@ server.modules = ( ) server.document-root = "/var/www/html" -server.error-handler-404 = "pihole/index.php" +server.error-handler-404 = "/pihole/index.php" server.upload-dirs = ( "/var/cache/lighttpd/uploads" ) server.errorlog = "/var/log/lighttpd/error.log" server.pid-file = "/var/run/lighttpd.pid" From 23b3a9a650d49afd8a1fd945f88415c659cce0d9 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 1 Jul 2019 09:44:20 +0200 Subject: [PATCH 143/207] Add DISTINCT to the view\'s SELECT queries to avoid domain duplication in case they are enabled in multiple groups at the same time. Signed-off-by: DL6ER --- advanced/Templates/gravity.db.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 3cbc7b8f..91a38ef7 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -92,7 +92,7 @@ CREATE VIEW vw_gravity AS SELECT domain FROM gravity WHERE domain NOT IN (SELECT domain from vw_whitelist); -CREATE VIEW vw_whitelist AS SELECT domain +CREATE VIEW vw_whitelist AS SELECT DISTINCT domain FROM whitelist LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id LEFT JOIN "group" ON "group".id = whitelist_by_group.group_id @@ -104,7 +104,7 @@ CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_blacklist AS SELECT domain +CREATE VIEW vw_blacklist AS SELECT DISTINCT domain FROM blacklist LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id LEFT JOIN "group" ON "group".id = blacklist_by_group.group_id @@ -116,7 +116,7 @@ CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_regex AS SELECT domain +CREATE VIEW vw_regex AS SELECT DISTINCT domain FROM regex LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id LEFT JOIN "group" ON "group".id = regex_by_group.group_id @@ -128,7 +128,7 @@ CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; -CREATE VIEW vw_adlist AS SELECT address +CREATE VIEW vw_adlist AS SELECT DISTINCT address FROM adlist LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id LEFT JOIN "group" ON "group".id = adlist_by_group.group_id From 9fedafed1595a2cb5fd9250b1e00e8088912e0da Mon Sep 17 00:00:00 2001 From: Jeroen Baert <3607063+Forceflow@users.noreply.github.com> Date: Wed, 3 Jul 2019 04:28:23 +0200 Subject: [PATCH 144/207] Fix 404 error when browsing to pi.hole (without /admin) (for fedora) Signed-off-by: Jeroen Baert <3607063+Forceflow@users.noreply.github.com> --- advanced/lighttpd.conf.fedora | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora index 27bc33cc..4232c90f 100644 --- a/advanced/lighttpd.conf.fedora +++ b/advanced/lighttpd.conf.fedora @@ -28,7 +28,7 @@ server.modules = ( ) server.document-root = "/var/www/html" -server.error-handler-404 = "pihole/index.php" +server.error-handler-404 = "/pihole/index.php" server.upload-dirs = ( "/var/cache/lighttpd/uploads" ) server.errorlog = "/var/log/lighttpd/error.log" server.pid-file = "/var/run/lighttpd.pid" From 1fe3507bc5e5e76109eecd9bd025dd024abf858f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 3 Jul 2019 07:56:10 +0200 Subject: [PATCH 145/207] Adlists table was renamed to Adlist. Adapt further places in the code to match the new name. Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 2 +- advanced/Scripts/query.sh | 2 +- advanced/Scripts/webpage.sh | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index b31bbdc5..2bfeef2d 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1077,7 +1077,7 @@ show_db_entries() { } show_adlists() { - show_db_entries "Adlists" "SELECT * FROM adlists" "4 100 7 10 13 50" + show_db_entries "Adlists" "SELECT * FROM adlist" "4 100 7 10 13 50" } show_whitelist() { diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 42ea4395..af5ae0a8 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -201,7 +201,7 @@ fi # Get adlist file content as array if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then # Retrieve source URLs from gravity database - mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" + mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" fi # Print "Exact matches for" title diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 583579b6..f744d1b9 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -400,13 +400,13 @@ CustomizeAdLists() { address="${args[3]}" if [[ "${args[2]}" == "enable" ]]; then - sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 1 WHERE address = '${address}'" + sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'" elif [[ "${args[2]}" == "disable" ]]; then - sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 0 WHERE address = '${address}'" + sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'" elif [[ "${args[2]}" == "add" ]]; then - sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlists (address) VALUES ('${address}')" + sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address) VALUES ('${address}')" elif [[ "${args[2]}" == "del" ]]; then - sqlite3 "${gravityDBfile}" "DELETE FROM adlists WHERE address = '${address}'" + sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'" else echo "Not permitted" return 1 From 04abcb71f62acb12c60f79de61db510e29fcb42c Mon Sep 17 00:00:00 2001 From: MMotti Date: Wed, 3 Jul 2019 13:04:57 +0100 Subject: [PATCH 146/207] Fix potential invalid result for pihole -q Signed-off-by: MMotti --- advanced/Scripts/query.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 42ea4395..b7edaccf 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -41,7 +41,7 @@ scanList(){ # Iterate through each regexp and check whether it matches the domainQuery # If it does, print the matching regexp and continue looping # Input 1 - regexps | Input 2 - domainQuery - "regex" ) awk 'NR==FNR{regexps[$0]}{for (r in regexps)if($0 ~ r)print r}' \ + "regex" ) awk 'NR==FNR{regexps[$0];next}{for (r in regexps)if($0 ~ r)print r}' \ <(echo "${lists}") <(echo "${domain}") 2>/dev/null;; * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; esac From f1733f9c5d8a798b38e372abeba30c7ddc540a87 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 4 Jul 2019 13:11:46 -0700 Subject: [PATCH 147/207] Fetch adlists for the block page from gravity.db Signed-off-by: Mcat12 --- advanced/index.php | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/advanced/index.php b/advanced/index.php index 78135e1a..b44a725d 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -111,11 +111,30 @@ if (is_file("/etc/pihole/adlists.list")) { die("[ERROR] File not found: /etc/pihole/adlists.list"); } -// Get all URLs starting with "http" or "www" from adlists and re-index array numerically -$adlistsUrls = array_values(preg_grep("/(^http)|(^www)/i", file($adLists, FILE_IGNORE_NEW_LINES))); +// Get possible non-standard location of FTL's database +$FTLsettings = parse_ini_file("/etc/pihole/pihole-FTL.conf"); +if(isset($FTLsettings["GRAVITYDB"])) { + $gravityDBFile = $FTLsettings["GRAVITYDB"]; +} else { + $gravityDBFile = "/etc/pihole/gravity.db"; +} + +// Connect to gravity.db +try { + $db = new SQLite3($gravityDBFile, SQLITE3_OPEN_READONLY); +} catch (Exception $exception) { + die("[ERROR]: Failed to connect to gravity.db"); +} + +// Get all adlist addresses +$adlistResults = $db->query("SELECT address FROM vw_adlist"); +$adlistsUrls = array(); +while($row = $adlistResults->fetchArray()) { + array_push($adlistsUrls, $row[0]); +} if (empty($adlistsUrls)) - die("[ERROR]: There are no adlist URL's found within $adLists"); + die("[ERROR]: There are no adlists configured"); // Get total number of blocklists (Including Whitelist, Blacklist & Wildcard lists) $adlistsCount = count($adlistsUrls) + 3; From 8d9ff550d469002b4aef323203f6ea7fd356f033 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 4 Jul 2019 13:44:14 -0700 Subject: [PATCH 148/207] Fix blockpage error if whitelisted, blacklisted, or regex filtered Signed-off-by: Mcat12 --- advanced/Scripts/query.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 4fc82744..6d061ba5 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -115,7 +115,9 @@ scanDatabaseTable() { wbMatch=true # Print table name - echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" + if [[ -z "${blockpage}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" + fi # Loop over results and print them mapfile -t results <<< "${result}" @@ -159,7 +161,7 @@ if [[ "${#regexList[@]}" -ne 0 ]]; then # shellcheck disable=SC2001 echo "${str_result}" | sed 's/^/ /' else - echo "π Regex list" + echo "π .wildcard" exit 0 fi fi From 2b5033e732e93be8e03a8049f1e83caa28a6bd25 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Thu, 4 Jul 2019 13:49:39 -0700 Subject: [PATCH 149/207] Add missing spaces found by linter Signed-off-by: Mcat12 --- advanced/index.php | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/index.php b/advanced/index.php index b44a725d..ff13ec60 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -113,7 +113,7 @@ if (is_file("/etc/pihole/adlists.list")) { // Get possible non-standard location of FTL's database $FTLsettings = parse_ini_file("/etc/pihole/pihole-FTL.conf"); -if(isset($FTLsettings["GRAVITYDB"])) { +if (isset($FTLsettings["GRAVITYDB"])) { $gravityDBFile = $FTLsettings["GRAVITYDB"]; } else { $gravityDBFile = "/etc/pihole/gravity.db"; @@ -129,7 +129,7 @@ try { // Get all adlist addresses $adlistResults = $db->query("SELECT address FROM vw_adlist"); $adlistsUrls = array(); -while($row = $adlistResults->fetchArray()) { +while ($row = $adlistResults->fetchArray()) { array_push($adlistsUrls, $row[0]); } From 1dbe6c83c38e64d2fb1026d65524a377ac990102 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 5 Jul 2019 13:54:18 +0200 Subject: [PATCH 150/207] Add database upgrading mechanism for adding the audit table. Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 21 +++++++++++++++++++ .../database_migration/gravity/1_to_2.sql | 9 ++++++++ advanced/Templates/gravity.db.sql | 11 +++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 advanced/Scripts/database_migration/gravity-db.sh create mode 100644 advanced/Scripts/database_migration/gravity/1_to_2.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh new file mode 100644 index 00000000..2c5669f0 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090 + +# Pi-hole: A black hole for Internet advertisements +# (c) 2019 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# Updates gravity.db database +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +upgrade_gravityDB(){ + version=$(sqlite3 "$1" "SELECT "value" FROM "info" WHERE "property" = 'version';") + echo $version + case "$version" in + 1) + sqlite3 "$1" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" + ;; + esac +} diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql new file mode 100644 index 00000000..63b7f56f --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -0,0 +1,9 @@ +CREATE TABLE audit +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); + +UPDATE info SET value = 2 WHERE property = 'version'; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 91a38ef7..eef9ac80 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -80,13 +80,22 @@ CREATE TABLE gravity ( domain TEXT PRIMARY KEY ); + +CREATE TABLE audit +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); + CREATE TABLE info ( property TEXT PRIMARY KEY, value TEXT NOT NULL ); -INSERT INTO info VALUES("version","1"); +INSERT INTO info VALUES("version","2"); CREATE VIEW vw_gravity AS SELECT domain FROM gravity From 4f4a12bb40ab6f09a8651ad5ad908edc8c9b59b1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 5 Jul 2019 14:03:57 +0200 Subject: [PATCH 151/207] Upgrade database if necessary and store audit domains therein. Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 4 +- gravity.sh | 54 +++++++++++++------ 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 2c5669f0..492546be 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -11,8 +11,8 @@ # Please see LICENSE file for your rights under this license. upgrade_gravityDB(){ - version=$(sqlite3 "$1" "SELECT "value" FROM "info" WHERE "property" = 'version';") - echo $version + local version=$(sqlite3 "$1" "SELECT "value" FROM "info" WHERE "property" = 'version';") + case "$version" in 1) sqlite3 "$1" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" diff --git a/gravity.sh b/gravity.sh index 1a325ba7..724238fa 100755 --- a/gravity.sh +++ b/gravity.sh @@ -17,6 +17,8 @@ coltable="/opt/pihole/COL_TABLE" source "${coltable}" regexconverter="/opt/pihole/wildcard_regex_converter.sh" source "${regexconverter}" +readonly databaseMigrationScript="/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" +source "${databaseMigrationScript}" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" @@ -28,6 +30,7 @@ whitelistFile="${piholeDir}/whitelist.txt" blacklistFile="${piholeDir}/blacklist.txt" regexFile="${piholeDir}/regex.list" adListFile="${piholeDir}/adlists.list" +auditFile="${piholeDir}/audit.list" localList="${piholeDir}/local.list" VPNList="/etc/openvpn/ipp.txt" @@ -116,14 +119,27 @@ database_table_from_file() { declare -i rowid rowid=1 # Read file line by line - grep -v '^ *#' < "${source}" | while IFS= read -r domain - do - # Only add non-empty lines - if [[ ! -z "${domain}" ]]; then - echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" - rowid+=1 - fi - done + if [[ "${table}" == "audit" ]]; then + grep -v '^ *#' < "${source}" | while IFS= read -r domain + do + # Only add non-empty lines + if [[ ! -z "${domain}" ]]; then + # Audit table format + echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" + rowid+=1 + fi + done + else + grep -v '^ *#' < "${source}" | while IFS= read -r domain + do + # Only add non-empty lines + if [[ ! -z "${domain}" ]]; then + # White-, black-, and regexlist format + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" + rowid+=1 + fi + done + fi inputfile="${tmpFile}" fi # Store domains in database table specified by ${table} @@ -150,34 +166,40 @@ database_table_from_file() { # Migrate pre-v5.0 list files to database-based Pi-hole versions migrate_to_database() { # Create database file only if not present - if [ -e "${gravityDBfile}" ]; then - return 0 + if [ ! -e "${gravityDBfile}" ]; then + echo -e " ${INFO} Creating new gravity database" + generate_gravity_database fi - echo -e " ${INFO} Creating new gravity database" - generate_gravity_database + # Check if gravity database needs to be updated + upgrade_gravityDB "${gravityDBfile}" # Migrate list files to new database - if [[ -e "${adListFile}" ]]; then + if [ -e "${adListFile}" ]; then # Store adlist domains in database echo -e " ${INFO} Migrating content of ${adListFile} into new database" database_table_from_file "adlist" "${adListFile}" fi - if [[ -e "${blacklistFile}" ]]; then + if [ -e "${blacklistFile}" ]; then # Store blacklisted domains in database echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" database_table_from_file "blacklist" "${blacklistFile}" fi - if [[ -e "${whitelistFile}" ]]; then + if [ -e "${whitelistFile}" ]; then # Store whitelisted domains in database echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" database_table_from_file "whitelist" "${whitelistFile}" fi - if [[ -e "${regexFile}" ]]; then + if [ -e "${regexFile}" ]; then # Store regex domains in database echo -e " ${INFO} Migrating content of ${regexFile} into new database" database_table_from_file "regex" "${regexFile}" fi + if [ -e "${auditFile}" ]; then + # Store audit domains in database + echo -e " ${INFO} Migrating content of ${auditFile} into new database" + database_table_from_file "audit" "${auditFile}" + fi } # Determine if DNS resolution is available before proceeding From 0c8f5f12215eaceb370e5110d8383fe34a428b17 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 5 Jul 2019 14:06:05 +0200 Subject: [PATCH 152/207] Remove comment field from audit table Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity/1_to_2.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql index 63b7f56f..4e560df5 100644 --- a/advanced/Scripts/database_migration/gravity/1_to_2.sql +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -2,8 +2,7 @@ CREATE TABLE audit ( id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) ); UPDATE info SET value = 2 WHERE property = 'version'; From 5293beeb77665fc60503f7566352e183c31e4d04 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 5 Jul 2019 14:10:33 +0200 Subject: [PATCH 153/207] Update audit script to store domains in new database table. Signed-off-by: DL6ER --- advanced/Scripts/webpage.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index f744d1b9..9a6ddfc9 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -546,17 +546,17 @@ addAudit() { shift # skip "-a" shift # skip "audit" - for var in "$@" + for domain in "$@" do - echo "${var}" >> /etc/pihole/auditlog.list + # Insert only the domain here. The date_added field will be + # filled with its default value (date_added = current timestamp) + sqlite3 "${gravityDBfile}" "INSERT INTO \"audit\" (domain) VALUES ('${domain}');" done - chmod 644 /etc/pihole/auditlog.list } clearAudit() { - echo -n "" > /etc/pihole/auditlog.list - chmod 644 /etc/pihole/auditlog.list + sqlite3 "${gravityDBfile}" "DELETE FROM \"audit\";" } SetPrivacyLevel() { From 82476138c151eba69c223943eb7ef1aa60cfed78 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 5 Jul 2019 16:09:13 +0200 Subject: [PATCH 154/207] Instead of calling sqlite3 multiple times within a loop, we use the ability to add multiple rows within one INSERT clause. This is supported since sqlite3 3.7.11 (2012-03-20) and should be available on all systems. Signed-off-by: DL6ER --- advanced/Scripts/webpage.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 9a6ddfc9..3996f7d0 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -546,12 +546,15 @@ addAudit() { shift # skip "-a" shift # skip "audit" + local domains="('${1}')" + shift # skip first domain, as it has already been added for domain in "$@" do # Insert only the domain here. The date_added field will be # filled with its default value (date_added = current timestamp) - sqlite3 "${gravityDBfile}" "INSERT INTO \"audit\" (domain) VALUES ('${domain}');" + domains="${domains},('${domain}')" done + sqlite3 "${gravityDBfile}" "INSERT INTO \"audit\" (domain) VALUES ${domains};" } clearAudit() From 2fb4256f84b1de7b264c5bf62849c5b7c58399bd Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 5 Jul 2019 16:28:36 +0200 Subject: [PATCH 155/207] Rename table to "auditlist" Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity/1_to_2.sql | 2 +- advanced/Scripts/webpage.sh | 4 ++-- advanced/Templates/gravity.db.sql | 5 ++--- gravity.sh | 8 ++++---- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql index 4e560df5..ba051e25 100644 --- a/advanced/Scripts/database_migration/gravity/1_to_2.sql +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -1,4 +1,4 @@ -CREATE TABLE audit +CREATE TABLE auditlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 3996f7d0..91d35d98 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -554,12 +554,12 @@ addAudit() # filled with its default value (date_added = current timestamp) domains="${domains},('${domain}')" done - sqlite3 "${gravityDBfile}" "INSERT INTO \"audit\" (domain) VALUES ${domains};" + sqlite3 "${gravityDBfile}" "INSERT INTO \"auditlist\" (domain) VALUES ${domains};" } clearAudit() { - sqlite3 "${gravityDBfile}" "DELETE FROM \"audit\";" + sqlite3 "${gravityDBfile}" "DELETE FROM \"auditlist\";" } SetPrivacyLevel() { diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index eef9ac80..78999e7c 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -81,12 +81,11 @@ CREATE TABLE gravity domain TEXT PRIMARY KEY ); -CREATE TABLE audit +CREATE TABLE auditlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), - comment TEXT + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) ); CREATE TABLE info diff --git a/gravity.sh b/gravity.sh index 724238fa..09c66c4f 100755 --- a/gravity.sh +++ b/gravity.sh @@ -30,7 +30,7 @@ whitelistFile="${piholeDir}/whitelist.txt" blacklistFile="${piholeDir}/blacklist.txt" regexFile="${piholeDir}/regex.list" adListFile="${piholeDir}/adlists.list" -auditFile="${piholeDir}/audit.list" +auditFile="${piholeDir}/auditlog.list" localList="${piholeDir}/local.list" VPNList="/etc/openvpn/ipp.txt" @@ -119,12 +119,12 @@ database_table_from_file() { declare -i rowid rowid=1 # Read file line by line - if [[ "${table}" == "audit" ]]; then + if [[ "${table}" == "auditlist" ]]; then grep -v '^ *#' < "${source}" | while IFS= read -r domain do # Only add non-empty lines if [[ ! -z "${domain}" ]]; then - # Audit table format + # Auditlist table format echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" rowid+=1 fi @@ -198,7 +198,7 @@ migrate_to_database() { if [ -e "${auditFile}" ]; then # Store audit domains in database echo -e " ${INFO} Migrating content of ${auditFile} into new database" - database_table_from_file "audit" "${auditFile}" + database_table_from_file "auditlist" "${auditFile}" fi } From 0405aaa3dac94da58585ff9ba7b131d5fc7c490f Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 6 Jul 2019 09:32:41 +0200 Subject: [PATCH 156/207] Review comments and fixing stickler complaints. Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 10 ++++----- .../database_migration/gravity/1_to_2.sql | 2 +- advanced/Scripts/webpage.sh | 22 +++++++++++++++---- advanced/Templates/gravity.db.sql | 2 +- gravity.sh | 15 ++++++++----- 5 files changed, 34 insertions(+), 17 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 492546be..55411214 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -11,11 +11,11 @@ # Please see LICENSE file for your rights under this license. upgrade_gravityDB(){ - local version=$(sqlite3 "$1" "SELECT "value" FROM "info" WHERE "property" = 'version';") + local version + version=$(sqlite3 "$1" "SELECT "value" FROM "info" WHERE "property" = 'version';") - case "$version" in - 1) + if [[ "$version" == "1" ]]; then sqlite3 "$1" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" - ;; - esac + version=2 + fi } diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql index ba051e25..073eced5 100644 --- a/advanced/Scripts/database_migration/gravity/1_to_2.sql +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -1,4 +1,4 @@ -CREATE TABLE auditlist +CREATE TABLE domain_auditlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 91d35d98..516ea4e4 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -542,24 +542,38 @@ Teleporter() { php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-teleporter_${datetimestamp}.tar.gz" } +checkDomain() +{ + local domain validDomain + # Convert to lowercase + domain="${1,,}" + validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check + validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label + echo "${validDomain}" +} + addAudit() { shift # skip "-a" shift # skip "audit" - local domains="('${1}')" + local domains validDomain + domains="('$(checkDomain "${1}")')" shift # skip first domain, as it has already been added for domain in "$@" do # Insert only the domain here. The date_added field will be # filled with its default value (date_added = current timestamp) - domains="${domains},('${domain}')" + validDomain="$(checkDomain "${domain}")" + if [[ -n "${validDomain}" ]]; then + domains="${domains},('${domain}')" + fi done - sqlite3 "${gravityDBfile}" "INSERT INTO \"auditlist\" (domain) VALUES ${domains};" + sqlite3 "${gravityDBfile}" "INSERT INTO \"domain_auditlist\" (domain) VALUES ${domains};" } clearAudit() { - sqlite3 "${gravityDBfile}" "DELETE FROM \"auditlist\";" + sqlite3 "${gravityDBfile}" "DELETE FROM \"domain_auditlist\";" } SetPrivacyLevel() { diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 78999e7c..113c035f 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -81,7 +81,7 @@ CREATE TABLE gravity domain TEXT PRIMARY KEY ); -CREATE TABLE auditlist +CREATE TABLE domain_auditlist ( id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, diff --git a/gravity.sh b/gravity.sh index 09c66c4f..3a8afc7a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -115,25 +115,28 @@ database_table_from_file() { inputfile="${source}" else # Apply format for white-, blacklist, regex, and adlist tables - local rowid - declare -i rowid - rowid=1 # Read file line by line if [[ "${table}" == "auditlist" ]]; then + local rowid + declare -i rowid + rowid=1 grep -v '^ *#' < "${source}" | while IFS= read -r domain do # Only add non-empty lines - if [[ ! -z "${domain}" ]]; then + if [[ -n "${domain}" ]]; then # Auditlist table format echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" rowid+=1 fi done else + local rowid + declare -i rowid + rowid=1 grep -v '^ *#' < "${source}" | while IFS= read -r domain do # Only add non-empty lines - if [[ ! -z "${domain}" ]]; then + if [[ -n "${domain}" ]]; then # White-, black-, and regexlist format echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" rowid+=1 @@ -198,7 +201,7 @@ migrate_to_database() { if [ -e "${auditFile}" ]; then # Store audit domains in database echo -e " ${INFO} Migrating content of ${auditFile} into new database" - database_table_from_file "auditlist" "${auditFile}" + database_table_from_file "domain_auditlist" "${auditFile}" fi } From efe8216445fbb7345a381f91fed3a3fefd5f5117 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 6 Jul 2019 09:45:07 +0200 Subject: [PATCH 157/207] Fix further stickler complaint. Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 55411214..46ff0a72 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -12,7 +12,7 @@ upgrade_gravityDB(){ local version - version=$(sqlite3 "$1" "SELECT "value" FROM "info" WHERE "property" = 'version';") + version="$(sqlite3 "$1" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" if [[ "$version" == "1" ]]; then sqlite3 "$1" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" From c48b03584f315d20f8f05561709e59bd462ebb28 Mon Sep 17 00:00:00 2001 From: bcambl Date: Sat, 6 Jul 2019 10:57:57 -0600 Subject: [PATCH 158/207] update Fedora Dockerfile to 30 Signed-off-by: bcambl --- test/fedora.Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/fedora.Dockerfile b/test/fedora.Dockerfile index 8c27bbcc..a06fc007 100644 --- a/test/fedora.Dockerfile +++ b/test/fedora.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:29 +FROM fedora:30 ENV GITDIR /etc/.pihole ENV SCRIPTDIR /opt/pihole From ec850dc82090683fd041673f489585e31e99569f Mon Sep 17 00:00:00 2001 From: bcambl Date: Sat, 6 Jul 2019 10:58:19 -0600 Subject: [PATCH 159/207] add chkconfig to INSTALLER_DEPS (CentOS/Fedora) chkconfig is a dependency of spawn-fcgi which is a dependency of lighttpd which is installed via PIHOLE_WEB_DEPS in phase 2 adding chkconfig to INSTALLER_DEPS to ensure /etc/init.d is present during the installation prompts (phase 1) Signed-off-by: bcambl --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1e87b943..a4adac94 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -283,7 +283,7 @@ elif is_command rpm ; then UPDATE_PKG_CACHE=":" PKG_INSTALL=("${PKG_MANAGER}" install -y) PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" - INSTALLER_DEPS=(dialog git iproute newt procps-ng which) + INSTALLER_DEPS=(dialog git iproute newt procps-ng which chkconfig) PIHOLE_DEPS=(bind-utils cronie curl findutils nmap-ncat sudo unzip wget libidn2 psmisc sqlite libcap) PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo) LIGHTTPD_USER="lighttpd" From 218476fab087a3fbd07170249a4b1371fb5f92f6 Mon Sep 17 00:00:00 2001 From: bcambl Date: Sat, 6 Jul 2019 11:06:08 -0600 Subject: [PATCH 160/207] ensure installation dependencies for FTL tests which rely on /etc/init.d Signed-off-by: bcambl --- test/test_automated_install.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 282c627d..4bfb0f6a 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -486,6 +486,13 @@ def test_FTL_download_aarch64_no_errors(Pihole): ''' confirms only aarch64 package is downloaded for FTL engine ''' + # mock whiptail answers and ensure installer dependencies + mock_command('whiptail', {'*': ('', '0')}, Pihole) + Pihole.run(''' + source /opt/pihole/basic-install.sh + distro_check + install_dependent_packages ${INSTALLER_DEPS[@]} + ''') download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh binary="pihole-FTL-aarch64-linux-gnu" @@ -501,6 +508,13 @@ def test_FTL_download_unknown_fails_no_errors(Pihole): ''' confirms unknown binary is not downloaded for FTL engine ''' + # mock whiptail answers and ensure installer dependencies + mock_command('whiptail', {'*': ('', '0')}, Pihole) + Pihole.run(''' + source /opt/pihole/basic-install.sh + distro_check + install_dependent_packages ${INSTALLER_DEPS[@]} + ''') download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh binary="pihole-FTL-mips" @@ -519,6 +533,13 @@ def test_FTL_download_binary_unset_no_errors(Pihole): ''' confirms unset binary variable does not download FTL engine ''' + # mock whiptail answers and ensure installer dependencies + mock_command('whiptail', {'*': ('', '0')}, Pihole) + Pihole.run(''' + source /opt/pihole/basic-install.sh + distro_check + install_dependent_packages ${INSTALLER_DEPS[@]} + ''') download_binary = Pihole.run(''' source /opt/pihole/basic-install.sh create_pihole_user From acc50b709efab4a88e2c1f2e7940b5708a47b6aa Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Jul 2019 10:33:08 +0200 Subject: [PATCH 161/207] Only migrate files once (domain and adlist lists druing initial creation of gravity.db auditlog.list on database upgrade from version 1 to 2. Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 9 ++ advanced/Templates/gravity.db.sql | 9 +- gravity.sh | 87 ++++++++----------- 3 files changed, 46 insertions(+), 59 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 46ff0a72..714676f4 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -15,7 +15,16 @@ upgrade_gravityDB(){ version="$(sqlite3 "$1" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" if [[ "$version" == "1" ]]; then + # This migration script upgrades the gravity.db file by + # adding the domain_auditlist table sqlite3 "$1" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" version=2 + + # Store audit domains in database table + if [ -e "${auditFile}" ]; then + echo -e " ${INFO} Migrating content of ${auditFile} into new database" + # database_table_from_file is defined in gravity.sh + database_table_from_file "domain_auditlist" "${auditFile}" + fi fi } diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 113c035f..09d581f0 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -81,20 +81,13 @@ CREATE TABLE gravity domain TEXT PRIMARY KEY ); -CREATE TABLE domain_auditlist -( - id INTEGER PRIMARY KEY AUTOINCREMENT, - domain TEXT UNIQUE NOT NULL, - date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) -); - CREATE TABLE info ( property TEXT PRIMARY KEY, value TEXT NOT NULL ); -INSERT INTO info VALUES("version","2"); +INSERT INTO info VALUES("version","1"); CREATE VIEW vw_gravity AS SELECT domain FROM gravity diff --git a/gravity.sh b/gravity.sh index 3a8afc7a..e2f2a9a8 100755 --- a/gravity.sh +++ b/gravity.sh @@ -17,8 +17,7 @@ coltable="/opt/pihole/COL_TABLE" source "${coltable}" regexconverter="/opt/pihole/wildcard_regex_converter.sh" source "${regexconverter}" -readonly databaseMigrationScript="/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" -source "${databaseMigrationScript}" +source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" basename="pihole" PIHOLE_COMMAND="/usr/local/bin/${basename}" @@ -116,33 +115,23 @@ database_table_from_file() { else # Apply format for white-, blacklist, regex, and adlist tables # Read file line by line - if [[ "${table}" == "auditlist" ]]; then - local rowid - declare -i rowid - rowid=1 - grep -v '^ *#' < "${source}" | while IFS= read -r domain - do - # Only add non-empty lines - if [[ -n "${domain}" ]]; then + local rowid + declare -i rowid + rowid=1 + grep -v '^ *#' < "${source}" | while IFS= read -r domain + do + # Only add non-empty lines + if [[ -n "${domain}" ]]; then + if [[ "${table}" == "auditlist" ]]; then # Auditlist table format echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" - rowid+=1 - fi - done - else - local rowid - declare -i rowid - rowid=1 - grep -v '^ *#' < "${source}" | while IFS= read -r domain - do - # Only add non-empty lines - if [[ -n "${domain}" ]]; then + else # White-, black-, and regexlist format echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" - rowid+=1 fi - done - fi + rowid+=1 + fi + done inputfile="${tmpFile}" fi # Store domains in database table specified by ${table} @@ -170,39 +159,35 @@ database_table_from_file() { migrate_to_database() { # Create database file only if not present if [ ! -e "${gravityDBfile}" ]; then + # Create new database file - note that this will be created in version 1 echo -e " ${INFO} Creating new gravity database" generate_gravity_database + + # Migrate list files to new database + if [ -e "${adListFile}" ]; then + # Store adlist domains in database + echo -e " ${INFO} Migrating content of ${adListFile} into new database" + database_table_from_file "adlist" "${adListFile}" + fi + if [ -e "${blacklistFile}" ]; then + # Store blacklisted domains in database + echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" + database_table_from_file "blacklist" "${blacklistFile}" + fi + if [ -e "${whitelistFile}" ]; then + # Store whitelisted domains in database + echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" + database_table_from_file "whitelist" "${whitelistFile}" + fi + if [ -e "${regexFile}" ]; then + # Store regex domains in database + echo -e " ${INFO} Migrating content of ${regexFile} into new database" + database_table_from_file "regex" "${regexFile}" + fi fi # Check if gravity database needs to be updated upgrade_gravityDB "${gravityDBfile}" - - # Migrate list files to new database - if [ -e "${adListFile}" ]; then - # Store adlist domains in database - echo -e " ${INFO} Migrating content of ${adListFile} into new database" - database_table_from_file "adlist" "${adListFile}" - fi - if [ -e "${blacklistFile}" ]; then - # Store blacklisted domains in database - echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" - database_table_from_file "blacklist" "${blacklistFile}" - fi - if [ -e "${whitelistFile}" ]; then - # Store whitelisted domains in database - echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" - database_table_from_file "whitelist" "${whitelistFile}" - fi - if [ -e "${regexFile}" ]; then - # Store regex domains in database - echo -e " ${INFO} Migrating content of ${regexFile} into new database" - database_table_from_file "regex" "${regexFile}" - fi - if [ -e "${auditFile}" ]; then - # Store audit domains in database - echo -e " ${INFO} Migrating content of ${auditFile} into new database" - database_table_from_file "domain_auditlist" "${auditFile}" - fi } # Determine if DNS resolution is available before proceeding From be3e198f9a83989a6db71011a294cdce0ef7f56c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Jul 2019 10:46:20 +0200 Subject: [PATCH 162/207] Address linting errors. Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 8 +++++--- gravity.sh | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 714676f4..294fd32a 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -11,13 +11,15 @@ # Please see LICENSE file for your rights under this license. upgrade_gravityDB(){ - local version - version="$(sqlite3 "$1" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" + local database auditFile version + database="${1}" + auditFile="${2}" + version="$(sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" if [[ "$version" == "1" ]]; then # This migration script upgrades the gravity.db file by # adding the domain_auditlist table - sqlite3 "$1" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" + sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" version=2 # Store audit domains in database table diff --git a/gravity.sh b/gravity.sh index e2f2a9a8..89f77ce0 100755 --- a/gravity.sh +++ b/gravity.sh @@ -17,6 +17,7 @@ coltable="/opt/pihole/COL_TABLE" source "${coltable}" regexconverter="/opt/pihole/wildcard_regex_converter.sh" source "${regexconverter}" +# shellcheck disable=SC1091 source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" basename="pihole" @@ -187,7 +188,7 @@ migrate_to_database() { fi # Check if gravity database needs to be updated - upgrade_gravityDB "${gravityDBfile}" + upgrade_gravityDB "${gravityDBfile}" "${auditFile}" } # Determine if DNS resolution is available before proceeding From 8382f4d7274e893e19dbe09de5dd090daa590c24 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 7 Jul 2019 21:21:56 +0200 Subject: [PATCH 163/207] Rename table to domain_audit and simplify subroutine addAudit(). Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 4 ++-- .../Scripts/database_migration/gravity/1_to_2.sql | 2 +- advanced/Scripts/webpage.sh | 14 +++++++++----- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 294fd32a..51a3480b 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -18,7 +18,7 @@ upgrade_gravityDB(){ if [[ "$version" == "1" ]]; then # This migration script upgrades the gravity.db file by - # adding the domain_auditlist table + # adding the domain_audit table sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" version=2 @@ -26,7 +26,7 @@ upgrade_gravityDB(){ if [ -e "${auditFile}" ]; then echo -e " ${INFO} Migrating content of ${auditFile} into new database" # database_table_from_file is defined in gravity.sh - database_table_from_file "domain_auditlist" "${auditFile}" + database_table_from_file "domain_audit" "${auditFile}" fi fi } diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql index 073eced5..90a48418 100644 --- a/advanced/Scripts/database_migration/gravity/1_to_2.sql +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -1,4 +1,4 @@ -CREATE TABLE domain_auditlist +CREATE TABLE domain_audit ( id INTEGER PRIMARY KEY AUTOINCREMENT, domain TEXT UNIQUE NOT NULL, diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 516ea4e4..918fde27 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -557,23 +557,27 @@ addAudit() shift # skip "-a" shift # skip "audit" local domains validDomain - domains="('$(checkDomain "${1}")')" - shift # skip first domain, as it has already been added + domains="" for domain in "$@" do # Insert only the domain here. The date_added field will be # filled with its default value (date_added = current timestamp) validDomain="$(checkDomain "${domain}")" if [[ -n "${validDomain}" ]]; then - domains="${domains},('${domain}')" + # Put comma in between () when there is + # more than one domains to be added + if [[ -n "${domains}" ]]; then + domains="${domains}," + fi + domains="${domains}('${domain}')" fi done - sqlite3 "${gravityDBfile}" "INSERT INTO \"domain_auditlist\" (domain) VALUES ${domains};" + sqlite3 "${gravityDBfile}" "INSERT INTO \"domain_audit\" (domain) VALUES ${domains};" } clearAudit() { - sqlite3 "${gravityDBfile}" "DELETE FROM \"domain_auditlist\";" + sqlite3 "${gravityDBfile}" "DELETE FROM \"domain_audit\";" } SetPrivacyLevel() { From e8e5d4afda44f8d46d546cc12140b0dcc980aa39 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Sun, 7 Jul 2019 18:10:39 -0700 Subject: [PATCH 164/207] Get file locations of FTL files from the config Instead of hardcoding the location of certain FTL files (`gravity.db`, `pihole-FTL.log`), read the configured location from FTL's config. The default location is used if no custom location has been configured. Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 2bfeef2d..b9bd6e8c 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -94,7 +94,35 @@ PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_SETUP_VARS_FILE="${PIHOLE_DIRECTORY}/setupVars.conf" -PIHOLE_GRAVITY_DB_FILE="${PIHOLE_DIRECTORY}/gravity.db" +PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole-FTL.conf" + +# Read the value of an FTL config key. The value is printed to stdout. +# +# Args: +# 1. The key to read +# 2. The default if the setting or config does not exist +get_ftl_conf_value() { + local key=$1 + local default=$2 + local value + + # Obtain key=... setting from pihole-FTL.conf + if [[ -e "$PIHOLE_FTL_CONF_FILE" ]]; then + # Constructed to return nothing when + # a) the setting is not present in the config file, or + # b) the setting is commented out (e.g. "#DBFILE=...") + value="$(sed -n -e "s/^\\s*$key=\\s*//p" ${PIHOLE_FTL_CONF_FILE})" + fi + + # Test for missing value. Use default value in this case. + if [[ -z "$value" ]]; then + value="$default" + fi + + echo "$value" +} + +PIHOLE_GRAVITY_DB_FILE=$(get_ftl_conf_value "GRAVITYDB" "${PIHOLE_DIRECTORY}/gravity.db") PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole" PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE" @@ -105,7 +133,7 @@ FTL_PORT="${RUN_DIRECTORY}/pihole-FTL.port" PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log" PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*" PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log" -PIHOLE_FTL_LOG="${LOG_DIRECTORY}/pihole-FTL.log" +PIHOLE_FTL_LOG=$(get_ftl_conf_value "LOGFILE" "${LOG_DIRECTORY}/pihole-FTL.log") PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access.log" PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error.log" From 3d3fc2947e5d848e7ea73bef7b6a34a9faa48091 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 8 Jul 2019 19:22:35 +0200 Subject: [PATCH 165/207] Review comments Signed-off-by: DL6ER --- advanced/Scripts/webpage.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 918fde27..227363f0 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -564,20 +564,22 @@ addAudit() # filled with its default value (date_added = current timestamp) validDomain="$(checkDomain "${domain}")" if [[ -n "${validDomain}" ]]; then - # Put comma in between () when there is + # Put comma in between domains when there is # more than one domains to be added + # SQL INSERT allows adding multiple rows at once using the format + ## INSERT INTO table (domain) VALUES ('abc.de'),('fgh.ij'),('klm.no'),('pqr.st'); if [[ -n "${domains}" ]]; then domains="${domains}," fi domains="${domains}('${domain}')" fi done - sqlite3 "${gravityDBfile}" "INSERT INTO \"domain_audit\" (domain) VALUES ${domains};" + sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};" } clearAudit() { - sqlite3 "${gravityDBfile}" "DELETE FROM \"domain_audit\";" + sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;" } SetPrivacyLevel() { From 054c7a2c050b416ac7de09e29351bdb69564430d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 8 Jul 2019 21:23:46 +0200 Subject: [PATCH 166/207] Create new table + view regex_whitelist + rename old regex table to regex_blacklist. This updates the gravity.db version to 3. Signed-off-by: DL6ER --- .../Scripts/database_migration/gravity-db.sh | 7 +++++ .../database_migration/gravity/2_to_3.sql | 30 +++++++++++++++++++ advanced/Scripts/list.sh | 21 ++++++------- advanced/Scripts/piholeDebug.sh | 11 +++++-- advanced/Scripts/query.sh | 2 +- gravity.sh | 9 +++--- 6 files changed, 62 insertions(+), 18 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/2_to_3.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 51a3480b..f37ce176 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -29,4 +29,11 @@ upgrade_gravityDB(){ database_table_from_file "domain_audit" "${auditFile}" fi fi + if [[ "$version" == "2" ]]; then + # This migration script upgrades the gravity.db file by + # renaming the regex table to regex_blacklist, and + # creating a new regex_whitelist table + corresponding linking table and views + sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/2_to_3.sql" + version=3 + fi } diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql new file mode 100644 index 00000000..e368592a --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql @@ -0,0 +1,30 @@ +PRAGMA FOREIGN_KEYS=OFF; + +ALTER TABLE regex RENAME TO regex_blacklist; + +CREATE TABLE regex_blacklist_by_group +( + regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_blacklist_id, group_id) +); + +INSERT INTO regex_blacklist_by_group SELECT * FROM regex_by_group; +DROP TABLE regex_by_group; +DROP VIEW vw_regex; +DROP TRIGGER tr_regex_update; + +CREATE VIEW vw_regex_blacklist AS SELECT DISTINCT domain + FROM regex + LEFT JOIN regex_blacklist_by_group ON regex_blacklist_by_group.regex_blacklist_id = regex_blacklist.id + LEFT JOIN "group" ON "group".id = regex_blacklist_by_group.group_id + WHERE regex_blacklist.enabled = 1 AND (regex_blacklist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY regex_blacklist.id; + +CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist + BEGIN + UPDATE regex_blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + + +UPDATE info SET value = 3 WHERE property = 'version'; diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index fa81348b..84acf4fc 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -32,12 +32,12 @@ helpFunc() { if [[ "${listType}" == "whitelist" ]]; then param="w" type="whitelist" - elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then + elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then param="-wild" type="wildcard blacklist" - elif [[ "${listType}" == "regex" ]]; then + elif [[ "${listType}" == "regex_blacklist" ]]; then param="-regex" - type="regex filter" + type="regex blacklist filter" else param="b" type="blacklist" @@ -58,7 +58,8 @@ Options: exit 0 } -EscapeRegexp() { +Escape +Regexp() { # This way we may safely insert an arbitrary # string in our regular expressions # This sed is intentionally executed in three steps to ease maintainability @@ -72,7 +73,7 @@ HandleOther() { # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then - if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then + if [[ "${listType}" == "regex_blacklist" && "${wildcard}" == false ]]; then validDomain="${domain}" else validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check @@ -88,9 +89,9 @@ HandleOther() { } ProcessDomainList() { - if [[ "${listType}" == "regex" ]]; then + if [[ "${listType}" == "regex_blacklist" ]]; then # Regex filter list - listname="regex filters" + listname="regex blacklist filters" else # Whitelist / Blacklist listname="${listType}" @@ -106,7 +107,7 @@ ProcessDomainList() { # if delmode then remove from desired list but do not add to the other if ${addmode}; then AddDomain "${dom}" "${listType}" - if [[ ! "${listType}" == "regex" ]]; then + if [[ ! "${listType}" == "regex_blacklist" ]]; then RemoveDomain "${dom}" "${listAlt}" fi else @@ -215,8 +216,8 @@ for var in "$@"; do case "${var}" in "-w" | "whitelist" ) listType="whitelist"; listAlt="blacklist";; "-b" | "blacklist" ) listType="blacklist"; listAlt="whitelist";; - "--wild" | "wildcard" ) listType="regex"; wildcard=true;; - "--regex" | "regex" ) listType="regex";; + "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;; + "--regex" | "regex" ) listType="regex_blacklist";; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 2bfeef2d..965250f0 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1088,8 +1088,12 @@ show_blacklist() { show_db_entries "Blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50" } -show_regexlist() { - show_db_entries "Regexlist" "SELECT * FROM regex" "4 100 7 10 13 50" +show_regexblacklist() { + show_db_entries "Regexblacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50" +} + +show_regexwhitelist() { + show_db_entries "Regexwhitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50" } analyze_gravity_list() { @@ -1268,7 +1272,8 @@ analyze_gravity_list show_adlists show_whitelist show_blacklist -show_regexlist +show_regexblacklist +show_regexwhitelist show_content_of_pihole_files parse_locale analyze_pihole_log diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 4fc82744..a587f238 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -133,7 +133,7 @@ scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" # Scan Regex table -mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex" 2> /dev/null) +mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_blacklist" 2> /dev/null) # If we have regexps to process if [[ "${#regexList[@]}" -ne 0 ]]; then diff --git a/gravity.sh b/gravity.sh index 89f77ce0..a379e248 100755 --- a/gravity.sh +++ b/gravity.sh @@ -183,7 +183,7 @@ migrate_to_database() { if [ -e "${regexFile}" ]; then # Store regex domains in database echo -e " ${INFO} Migrating content of ${regexFile} into new database" - database_table_from_file "regex" "${regexFile}" + database_table_from_file "regex_blacklist" "${regexFile}" fi fi @@ -591,9 +591,10 @@ gravity_Table_Count() { # Output count of blacklisted domains and regex filters gravity_ShowCount() { - gravity_Table_Count "blacklist" "blacklisted domains" - gravity_Table_Count "whitelist" "whitelisted domains" - gravity_Table_Count "regex" "regex filters" + gravity_Table_Count "blacklist" "exact blacklisted domains" + gravity_Table_Count "regex_blacklist" "regex blacklist filters" + gravity_Table_Count "whitelist" "exact whitelisted domains" + gravity_Table_Count "regex_whitelist" "regex whitelist filters" } # Parse list of domains into hosts format From f5121c64be312a0c4c900383d882f20cccdb8a6d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 8 Jul 2019 21:39:30 +0200 Subject: [PATCH 167/207] We should still add the regex lines (initially) to the regex table as the renaming will happen only after the importing. Signed-off-by: DL6ER --- gravity.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index a379e248..f9ecc1d1 100755 --- a/gravity.sh +++ b/gravity.sh @@ -182,8 +182,10 @@ migrate_to_database() { fi if [ -e "${regexFile}" ]; then # Store regex domains in database + # Important note: We need to add the domains to the "regex" table + # as it will only later be renamed to "regex_blacklist"! echo -e " ${INFO} Migrating content of ${regexFile} into new database" - database_table_from_file "regex_blacklist" "${regexFile}" + database_table_from_file "regex" "${regexFile}" fi fi From 0683842ec30081e57ce2e801ebcfec7c4f08f8ff Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 8 Jul 2019 21:43:49 +0200 Subject: [PATCH 168/207] Fix typo in 2->3 migration script. Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity/2_to_3.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql index e368592a..9b0bd412 100644 --- a/advanced/Scripts/database_migration/gravity/2_to_3.sql +++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql @@ -15,7 +15,7 @@ DROP VIEW vw_regex; DROP TRIGGER tr_regex_update; CREATE VIEW vw_regex_blacklist AS SELECT DISTINCT domain - FROM regex + FROM regex_blacklist LEFT JOIN regex_blacklist_by_group ON regex_blacklist_by_group.regex_blacklist_id = regex_blacklist.id LEFT JOIN "group" ON "group".id = regex_blacklist_by_group.group_id WHERE regex_blacklist.enabled = 1 AND (regex_blacklist_by_group.group_id IS NULL OR "group".enabled = 1) From b154dd5f0792ff804ae5f1b5f8c704c00787a3ca Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Mon, 8 Jul 2019 19:48:50 -0700 Subject: [PATCH 169/207] Quote calls to read FTL config Signed-off-by: Mcat12 --- advanced/Scripts/piholeDebug.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index b9bd6e8c..d1acb950 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -122,7 +122,7 @@ get_ftl_conf_value() { echo "$value" } -PIHOLE_GRAVITY_DB_FILE=$(get_ftl_conf_value "GRAVITYDB" "${PIHOLE_DIRECTORY}/gravity.db") +PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "GRAVITYDB" "${PIHOLE_DIRECTORY}/gravity.db")" PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole" PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE" @@ -133,7 +133,7 @@ FTL_PORT="${RUN_DIRECTORY}/pihole-FTL.port" PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log" PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*" PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log" -PIHOLE_FTL_LOG=$(get_ftl_conf_value "LOGFILE" "${LOG_DIRECTORY}/pihole-FTL.log") +PIHOLE_FTL_LOG="$(get_ftl_conf_value "LOGFILE" "${LOG_DIRECTORY}/pihole-FTL.log")" PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access.log" PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error.log" From 5ff90522002576d9a0202a7d458d1d38117659e1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 9 Jul 2019 11:41:44 +0200 Subject: [PATCH 170/207] Review comments Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity-db.sh | 7 +++++-- advanced/Scripts/webpage.sh | 5 +++-- gravity.sh | 7 +++---- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 51a3480b..65b42b95 100644 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -11,9 +11,12 @@ # Please see LICENSE file for your rights under this license. upgrade_gravityDB(){ - local database auditFile version + local database piholeDir auditFile version database="${1}" - auditFile="${2}" + piholeDir="${2}" + auditFile="${piholeDir}/auditlog.list" + + # Get database version version="$(sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" if [[ "$version" == "1" ]]; then diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 227363f0..9db10bda 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -560,8 +560,7 @@ addAudit() domains="" for domain in "$@" do - # Insert only the domain here. The date_added field will be - # filled with its default value (date_added = current timestamp) + # Check domain to be added. Only continue if it is valid validDomain="$(checkDomain "${domain}")" if [[ -n "${validDomain}" ]]; then # Put comma in between domains when there is @@ -574,6 +573,8 @@ addAudit() domains="${domains}('${domain}')" fi done + # Insert only the domain here. The date_added field will be + # filled with its default value (date_added = current timestamp) sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};" } diff --git a/gravity.sh b/gravity.sh index 89f77ce0..d7c66d68 100755 --- a/gravity.sh +++ b/gravity.sh @@ -30,7 +30,6 @@ whitelistFile="${piholeDir}/whitelist.txt" blacklistFile="${piholeDir}/blacklist.txt" regexFile="${piholeDir}/regex.list" adListFile="${piholeDir}/adlists.list" -auditFile="${piholeDir}/auditlog.list" localList="${piholeDir}/local.list" VPNList="/etc/openvpn/ipp.txt" @@ -123,8 +122,8 @@ database_table_from_file() { do # Only add non-empty lines if [[ -n "${domain}" ]]; then - if [[ "${table}" == "auditlist" ]]; then - # Auditlist table format + if [[ "${table}" == "domain_audit" ]]; then + # domain_audit table format (no enable or modified fields) echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}" else # White-, black-, and regexlist format @@ -188,7 +187,7 @@ migrate_to_database() { fi # Check if gravity database needs to be updated - upgrade_gravityDB "${gravityDBfile}" "${auditFile}" + upgrade_gravityDB "${gravityDBfile}" "${piholeDir}" } # Determine if DNS resolution is available before proceeding From 87f75c737a8d7eced48c5f61e2bb6581db1f4c2a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 10 Jul 2019 12:00:38 +0200 Subject: [PATCH 171/207] Review comments. Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 3 +-- advanced/Scripts/piholeDebug.sh | 16 ++++------------ 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 84acf4fc..e3dc552a 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -58,8 +58,7 @@ Options: exit 0 } -Escape -Regexp() { +EscapeRegexp() { # This way we may safely insert an arbitrary # string in our regular expressions # This sed is intentionally executed in three steps to ease maintainability diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 965250f0..0924e984 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1081,19 +1081,13 @@ show_adlists() { } show_whitelist() { - show_db_entries "Whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50" + show_db_entries "Exact whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50" + show_db_entries "Regex whitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50" } show_blacklist() { - show_db_entries "Blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50" -} - -show_regexblacklist() { - show_db_entries "Regexblacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50" -} - -show_regexwhitelist() { - show_db_entries "Regexwhitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50" + show_db_entries "Exact blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50" + show_db_entries "Regex blacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50" } analyze_gravity_list() { @@ -1272,8 +1266,6 @@ analyze_gravity_list show_adlists show_whitelist show_blacklist -show_regexblacklist -show_regexwhitelist show_content_of_pihole_files parse_locale analyze_pihole_log From 65fdbc85d5e1cbafb986da221783fc73ad972df8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 10 Jul 2019 12:01:38 +0200 Subject: [PATCH 172/207] Add timeout to migration script (2->3). Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity/2_to_3.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql index 9b0bd412..a2602c4a 100644 --- a/advanced/Scripts/database_migration/gravity/2_to_3.sql +++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql @@ -1,3 +1,5 @@ +.timeout 30000 + PRAGMA FOREIGN_KEYS=OFF; ALTER TABLE regex RENAME TO regex_blacklist; From 420f60b5c7748ccf939c104a77c9dbf30116f668 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Wed, 10 Jul 2019 11:56:39 +0200 Subject: [PATCH 173/207] Add timeout to migration script (1->2). Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity/1_to_2.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql index 90a48418..45b5fa02 100644 --- a/advanced/Scripts/database_migration/gravity/1_to_2.sql +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -1,3 +1,5 @@ +.timeout 30000 + CREATE TABLE domain_audit ( id INTEGER PRIMARY KEY AUTOINCREMENT, From fa8751f9ad89cbe1e9eb32784bdbb99e213ef390 Mon Sep 17 00:00:00 2001 From: Mark Drobnak Date: Wed, 10 Jul 2019 19:42:51 -0700 Subject: [PATCH 174/207] Fix error when checking if IP address is valid During install in `valid_ip`, we split up the IP address into octets to verify it is valid (each is <= 255). This validation was broken in #2743 when a variable usage was quoted where it should have stayed unquoted: ``` ./automated install/basic-install.sh: line 942: [[: 192.241.211.120: syntax error: invalid arithmetic operator (error token is ".241.211.120") ``` Due to this error, `127.0.0.1` would be used instead of the requested IP address. Also, this prevented the user from entering a custom DNS server as it would be marked as an invalid IP address. Signed-off-by: Mark Drobnak --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a4adac94..cb6783a2 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -934,7 +934,7 @@ valid_ip() { # and set the new one to a dot (period) IFS='.' # Put the IP into an array - ip=("${ip}") + ip=(${ip}) # Restore the IFS to what it was IFS=${OIFS} ## Evaluate each octet by checking if it's less than or equal to 255 (the max for each octet) From c156af020c017e0a0429724cba695a4d1984bba1 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Wed, 10 Jul 2019 19:52:17 -0700 Subject: [PATCH 175/207] Use suggested array creation to fix linter error Signed-off-by: Mcat12 --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index cb6783a2..17bdde31 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -934,7 +934,7 @@ valid_ip() { # and set the new one to a dot (period) IFS='.' # Put the IP into an array - ip=(${ip}) + read -r -a ip <<< "${ip}" # Restore the IFS to what it was IFS=${OIFS} ## Evaluate each octet by checking if it's less than or equal to 255 (the max for each octet) From 1d5755a4c2712156bd16d30fe61fcaef229714c9 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Wed, 10 Jul 2019 21:18:58 -0700 Subject: [PATCH 176/207] Add tests for valid_ip Signed-off-by: Mcat12 --- test/test_automated_install.py | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 4bfb0f6a..cce11857 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -700,3 +700,42 @@ def test_IPv6_ULA_GUA_test(Pihole): ''') expected_stdout = 'Found IPv6 ULA address, using it for blocking IPv6 ads' assert expected_stdout in detectPlatform.stdout + + +def test_validate_ip_valid(Pihole): + ''' + Given a valid IP address, valid_ip returns success + ''' + + output = Pihole.run(''' + source /opt/pihole/basic-install.sh + valid_ip "192.168.1.1" + ''') + + assert output.rc == 0 + + +def test_validate_ip_invalid_octet(Pihole): + ''' + Given an invalid IP address (large octet), valid_ip returns an error + ''' + + output = Pihole.run(''' + source /opt/pihole/basic-install.sh + valid_ip "1092.168.1.1" + ''') + + assert output.rc == 1 + + +def test_validate_ip_invalid_letters(Pihole): + ''' + Given an invalid IP address (contains letters), valid_ip returns an error + ''' + + output = Pihole.run(''' + source /opt/pihole/basic-install.sh + valid_ip "not an IP" + ''') + + assert output.rc == 1 From c3ec2e68adf68b8d4ae169385e1158754e8a67d3 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 12 Jul 2019 20:03:36 -0700 Subject: [PATCH 177/207] Remove the ZeusTracker blocklist from the defaults It is no longer served. Fixes #2843. Signed-off-by: Mcat12 --- automated install/basic-install.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 17bdde31..65fb7c2e 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1177,12 +1177,11 @@ chooseBlocklists() { mv "${adlistFile}" "${adlistFile}.old" fi # Let user select (or not) blocklists via a checklist - cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 7) + cmd=(whiptail --separate-output --checklist "Pi-hole relies on third party lists in order to block ads.\\n\\nYou can use the suggestions below, and/or add your own after installation\\n\\nTo deselect any list, use the arrow keys and spacebar" "${r}" "${c}" 6) # In an array, show the options available (all off by default): options=(StevenBlack "StevenBlack's Unified Hosts List" on MalwareDom "MalwareDomains" on Cameleon "Cameleon" on - ZeusTracker "ZeusTracker" on DisconTrack "Disconnect.me Tracking" on DisconAd "Disconnect.me Ads" on HostsFile "Hosts-file.net Ads" on) @@ -1205,7 +1204,6 @@ appendToListsFile() { StevenBlack ) echo "https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts" >> "${adlistFile}";; MalwareDom ) echo "https://mirror1.malwaredomains.com/files/justdomains" >> "${adlistFile}";; Cameleon ) echo "http://sysctl.org/cameleon/hosts" >> "${adlistFile}";; - ZeusTracker ) echo "https://zeustracker.abuse.ch/blocklist.php?download=domainblocklist" >> "${adlistFile}";; DisconTrack ) echo "https://s3.amazonaws.com/lists.disconnect.me/simple_tracking.txt" >> "${adlistFile}";; DisconAd ) echo "https://s3.amazonaws.com/lists.disconnect.me/simple_ad.txt" >> "${adlistFile}";; HostsFile ) echo "https://hosts-file.net/ad_servers.txt" >> "${adlistFile}";; @@ -1223,7 +1221,6 @@ installDefaultBlocklists() { appendToListsFile StevenBlack appendToListsFile MalwareDom appendToListsFile Cameleon - appendToListsFile ZeusTracker appendToListsFile DisconTrack appendToListsFile DisconAd appendToListsFile HostsFile From 38ff3431340d3d35307aa7c6b18aebad9d1ff7e6 Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 19 Jul 2019 17:35:21 -0700 Subject: [PATCH 178/207] Print an error message if the FTL release metadata download fails Signed-off-by: Mcat12 --- automated install/basic-install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 380b424f..3641b961 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -2404,6 +2404,7 @@ FTLcheckUpdate() { if ! FTLreleaseData=$(curl -sI https://github.com/pi-hole/FTL/releases/latest); then # There was an issue while retrieving the latest version + printf " %b Failed to retrieve latest FTL release metadata" "${CROSS}" return 3 fi From 3ebd43ebf00cd59b7ff8eef84885fd952a391acf Mon Sep 17 00:00:00 2001 From: Mcat12 Date: Fri, 19 Jul 2019 17:39:00 -0700 Subject: [PATCH 179/207] Remove outdated adlists.list check and fix empty adlists error message Signed-off-by: Mcat12 --- advanced/index.php | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/advanced/index.php b/advanced/index.php index ff13ec60..62e45091 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -102,15 +102,6 @@ if ($blocklistglob === array()) { die("[ERROR] There are no domain lists generated lists within /etc/pihole/! Please update gravity by running pihole -g, or repair Pi-hole using pihole -r."); } -// Set location of adlists file -if (is_file("/etc/pihole/adlists.list")) { - $adLists = "/etc/pihole/adlists.list"; -} elseif (is_file("/etc/pihole/adlists.default")) { - $adLists = "/etc/pihole/adlists.default"; -} else { - die("[ERROR] File not found: /etc/pihole/adlists.list"); -} - // Get possible non-standard location of FTL's database $FTLsettings = parse_ini_file("/etc/pihole/pihole-FTL.conf"); if (isset($FTLsettings["GRAVITYDB"])) { @@ -134,7 +125,7 @@ while ($row = $adlistResults->fetchArray()) { } if (empty($adlistsUrls)) - die("[ERROR]: There are no adlists configured"); + die("[ERROR]: There are no adlists enabled"); // Get total number of blocklists (Including Whitelist, Blacklist & Wildcard lists) $adlistsCount = count($adlistsUrls) + 3; From 96031214c6a0d797015b4461901cf1059edbc00a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2019 19:35:16 +0200 Subject: [PATCH 180/207] Add support for whitelist regex filter management via CLI. Signed-off-by: DL6ER --- .../database_migration/gravity/2_to_3.sql | 29 +++++++++++++++++++ advanced/Scripts/list.sh | 13 +++++++-- pihole | 2 ++ 3 files changed, 41 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql index a2602c4a..d7997936 100644 --- a/advanced/Scripts/database_migration/gravity/2_to_3.sql +++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql @@ -28,5 +28,34 @@ CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist UPDATE regex_blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; END; +CREATE TABLE regex_whitelist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); + +CREATE TABLE regex_whitelist_by_group +( + regex_id INTEGER NOT NULL REFERENCES regex_whitelist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_id, group_id) +); + +CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain + FROM regex_whitelist + LEFT JOIN regex_whitelist_by_group ON regex_whitelist_by_group.regex_id = regex_whitelist.id + LEFT JOIN "group" ON "group".id = regex_whitelist_by_group.group_id + WHERE regex_whitelist.enabled = 1 AND (regex_whitelist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY regex_whitelist.id; + +CREATE TRIGGER tr_regex_whitelist_update AFTER UPDATE ON regex_whitelist + BEGIN + UPDATE regex_whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + UPDATE info SET value = 3 WHERE property = 'version'; diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index e3dc552a..4ef86407 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -38,6 +38,9 @@ helpFunc() { elif [[ "${listType}" == "regex_blacklist" ]]; then param="-regex" type="regex blacklist filter" + elif [[ "${listType}" == "regex_blacklist" ]]; then + param="-whiteregex" + type="regex whitelist filter" else param="b" type="blacklist" @@ -89,8 +92,11 @@ HandleOther() { ProcessDomainList() { if [[ "${listType}" == "regex_blacklist" ]]; then - # Regex filter list + # Regex black filter list listname="regex blacklist filters" + elif [[ "${listType}" == "regex_whitelist" ]]; then + # Regex white filter list + listname="regex whitelist filters" else # Whitelist / Blacklist listname="${listType}" @@ -106,7 +112,7 @@ ProcessDomainList() { # if delmode then remove from desired list but do not add to the other if ${addmode}; then AddDomain "${dom}" "${listType}" - if [[ ! "${listType}" == "regex_blacklist" ]]; then + if [[ ! "${listType}" == "regex_"*"list" ]]; then RemoveDomain "${dom}" "${listAlt}" fi else @@ -173,7 +179,7 @@ Displaylist() { data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)" if [[ -z $data ]]; then - echo -e "Not showing empty ${listname}" + echo -e "Not showing empty list" else echo -e "Displaying ${listname}:" count=1 @@ -217,6 +223,7 @@ for var in "$@"; do "-b" | "blacklist" ) listType="blacklist"; listAlt="whitelist";; "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;; "--regex" | "regex" ) listType="regex_blacklist";; + "--whiteregex" | "whiteregex" ) listType="regex_whitelist";; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; diff --git a/pihole b/pihole index 9fa65a8f..411b5791 100755 --- a/pihole +++ b/pihole @@ -377,6 +377,7 @@ Whitelist/Blacklist Options: -b, blacklist Blacklist domain(s) --wild, wildcard Wildcard blacklist domain(s) --regex, regex Regex blacklist domains(s) + --whiteregex Regex whitelist domains(s) Add '-h' for more info on whitelist/blacklist usage Debugging Options: @@ -438,6 +439,7 @@ case "${1}" in "-b" | "blacklist" ) listFunc "$@";; "--wild" | "wildcard" ) listFunc "$@";; "--regex" | "regex" ) listFunc "$@";; + "--whiteregex" | "whiteregex" ) listFunc "$@";; "-d" | "debug" ) debugFunc "$@";; "-f" | "flush" ) flushFunc "$@";; "-up" | "updatePihole" ) updatePiholeFunc "$@";; From 0d28dce326facddc13f02fe80903f78ea732736a Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2019 20:18:15 +0200 Subject: [PATCH 181/207] Print group table contents in debug log. Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 0924e984..b2533eb0 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1076,18 +1076,27 @@ show_db_entries() { IFS="$OLD_IFS" } +show_groups() { + show_db_entries "Groups" "SELECT * FROM \"group\"" "4 100 7 10 13 50" +} + show_adlists() { show_db_entries "Adlists" "SELECT * FROM adlist" "4 100 7 10 13 50" + show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 100 7 10 13 50" } show_whitelist() { show_db_entries "Exact whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50" show_db_entries "Regex whitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50" + show_db_entries "Exact whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 100 7 10 13 50" + show_db_entries "Regex whitelist groups" "SELECT * FROM whitelist_by_group" "4 100 7 10 13 50" } show_blacklist() { show_db_entries "Exact blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50" show_db_entries "Regex blacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50" + show_db_entries "Exact blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 100 7 10 13 50" + show_db_entries "Regex blacklist groups" "SELECT * FROM blacklist_by_group" "4 100 7 10 13 50" } analyze_gravity_list() { @@ -1263,6 +1272,7 @@ process_status parse_setup_vars check_x_headers analyze_gravity_list +show_groups show_adlists show_whitelist show_blacklist From 0692be9bae12a8a96f582a2dc15968282df2e8f2 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2019 20:59:52 +0200 Subject: [PATCH 182/207] Fix small mistake in 2->3 upgrade script. Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity/2_to_3.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql index d7997936..e121f28c 100644 --- a/advanced/Scripts/database_migration/gravity/2_to_3.sql +++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql @@ -40,14 +40,14 @@ CREATE TABLE regex_whitelist CREATE TABLE regex_whitelist_by_group ( - regex_id INTEGER NOT NULL REFERENCES regex_whitelist (id), + regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id), group_id INTEGER NOT NULL REFERENCES "group" (id), - PRIMARY KEY (regex_id, group_id) + PRIMARY KEY (regex_whitelist_id, group_id) ); CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain FROM regex_whitelist - LEFT JOIN regex_whitelist_by_group ON regex_whitelist_by_group.regex_id = regex_whitelist.id + LEFT JOIN regex_whitelist_by_group ON regex_whitelist_by_group.regex_whitelist_id = regex_whitelist.id LEFT JOIN "group" ON "group".id = regex_whitelist_by_group.group_id WHERE regex_whitelist.enabled = 1 AND (regex_whitelist_by_group.group_id IS NULL OR "group".enabled = 1) ORDER BY regex_whitelist.id; From 40d0caa70b642382b2ca35344e6da7fd6c452432 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2019 21:03:42 +0200 Subject: [PATCH 183/207] Add undocumented --whitewild option that does the same --wild does for the whitelist. Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 3 ++- pihole | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 4ef86407..0183a9e2 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -75,7 +75,7 @@ HandleOther() { # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then - if [[ "${listType}" == "regex_blacklist" && "${wildcard}" == false ]]; then + if [[ ( "${listType}" == "regex_blacklist" || "${listType}" == "regex_whitelist" ) && "${wildcard}" == false ]]; then validDomain="${domain}" else validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check @@ -224,6 +224,7 @@ for var in "$@"; do "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;; "--regex" | "regex" ) listType="regex_blacklist";; "--whiteregex" | "whiteregex" ) listType="regex_whitelist";; + "--whitewild" | "whitewild" ) listType="regex_whitelist"; wildcard=true;; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; diff --git a/pihole b/pihole index 411b5791..b3260d83 100755 --- a/pihole +++ b/pihole @@ -440,6 +440,7 @@ case "${1}" in "--wild" | "wildcard" ) listFunc "$@";; "--regex" | "regex" ) listFunc "$@";; "--whiteregex" | "whiteregex" ) listFunc "$@";; + "--whitewild" | "whitewild" ) listFunc "$@";; "-d" | "debug" ) debugFunc "$@";; "-f" | "flush" ) flushFunc "$@";; "-up" | "updatePihole" ) updatePiholeFunc "$@";; From 6f58d58cae7ab35a523c39e268ecac2e562d16a1 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 22 Jul 2019 22:26:27 +0200 Subject: [PATCH 184/207] Add --whitewild to help texts and man pages. Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 11 +++++++---- manpages/pihole.8 | 14 ++++++++++++-- pihole | 3 ++- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 0183a9e2..f0cf4701 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -32,15 +32,18 @@ helpFunc() { if [[ "${listType}" == "whitelist" ]]; then param="w" type="whitelist" - elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then - param="-wild" - type="wildcard blacklist" elif [[ "${listType}" == "regex_blacklist" ]]; then param="-regex" type="regex blacklist filter" - elif [[ "${listType}" == "regex_blacklist" ]]; then + elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then + param="-wild" + type="wildcard blacklist" + elif [[ "${listType}" == "regex_whitelist" ]]; then param="-whiteregex" type="regex whitelist filter" + elif [[ "${listType}" == "regex_whitelist" && "${wildcard}" == true ]]; then + param="-whitewild" + type="wildcard whitelist" else param="b" type="blacklist" diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 065280c7..11923392 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -66,14 +66,24 @@ Available commands and options: Adds or removes specified domain or domains to the blacklist .br +\fB--regex, regex\fR [options] [ ] +.br + Add or removes specified regex filter to the regex blacklist +.br + +\fB--whiteregex\fR [options] [ ] +.br + Add or removes specified regex filter to the regex whitelist +.br + \fB--wild, wildcard\fR [options] [ ] .br Add or removes specified domain to the wildcard blacklist .br -\fB--regex, regex\fR [options] [ ] +\fB--whitewild\fR [options] [ ] .br - Add or removes specified regex filter to the regex blacklist + Add or removes specified domain to the wildcard whitelist .br (Whitelist/Blacklist manipulation options): diff --git a/pihole b/pihole index b3260d83..5d373a47 100755 --- a/pihole +++ b/pihole @@ -375,9 +375,10 @@ Add '-h' after specific commands for more information on usage Whitelist/Blacklist Options: -w, whitelist Whitelist domain(s) -b, blacklist Blacklist domain(s) - --wild, wildcard Wildcard blacklist domain(s) --regex, regex Regex blacklist domains(s) --whiteregex Regex whitelist domains(s) + --wild, wildcard Wildcard blacklist domain(s) + --whitewild Wildcard whitelist domain(s) Add '-h' for more info on whitelist/blacklist usage Debugging Options: From ecd6817aaf89e1c5012c9bbf0ac116ff6a79c4ab Mon Sep 17 00:00:00 2001 From: "B. Olausson" Date: Mon, 29 Jul 2019 18:34:00 +0200 Subject: [PATCH 185/207] This change fixes issue #145 "stty: standard input: Inappropriate ioctl for device ".It checks if a real terminal exist, if not it sets the screen size to a fixed value. This helps to avoid nasty and unnecessary logs when running "pihole -up" via e.g. cron. Signed-off-by: B. Olausson --- automated install/basic-install.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 5b81f691..9891fd9d 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -84,8 +84,13 @@ if [ -z "${USER}" ]; then fi -# Find the rows and columns will default to 80x24 if it can not be detected -screen_size=$(stty size || printf '%d %d' 24 80) +# Check if we are running on a real terminal and find the rows and columns +# If there is no real terminal, we will default to 80x24 +if [ -t 0 ] ; then + screen_size=$(stty size) +else + screen_size="24 80" +fi # Set rows variable to contain first number printf -v rows '%d' "${screen_size%% *}" # Set columns variable to contain second number From 63230cb72dbe4d7201aa68c6a5f3abe898f8de2d Mon Sep 17 00:00:00 2001 From: Andreas Date: Sun, 4 Aug 2019 21:21:08 +0200 Subject: [PATCH 186/207] quick fix for when dig also returns a CNAME Signed-off-by: ryrun --- gravity.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gravity.sh b/gravity.sh index d7c66d68..14b32827 100755 --- a/gravity.sh +++ b/gravity.sh @@ -349,7 +349,7 @@ gravity_DownloadBlocklistFromUrl() { else printf -v port "%s" "${PIHOLE_DNS_1#*#}" fi - ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}") + ip=$(dig "@${ip_addr}" -p "${port}" +short "${domain}" | tail -1) if [[ $(echo "${url}" | awk -F '://' '{print $1}') = "https" ]]; then port=443; else port=80 From a95b4734170c8a401a698cd03665546f3269396e Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 5 Aug 2019 20:56:01 +0200 Subject: [PATCH 187/207] Rearranage if statements to ensure the proper output is shown for wildcard-style filters. Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index f0cf4701..31822c9a 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -32,18 +32,18 @@ helpFunc() { if [[ "${listType}" == "whitelist" ]]; then param="w" type="whitelist" - elif [[ "${listType}" == "regex_blacklist" ]]; then - param="-regex" - type="regex blacklist filter" elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then param="-wild" type="wildcard blacklist" - elif [[ "${listType}" == "regex_whitelist" ]]; then - param="-whiteregex" - type="regex whitelist filter" + elif [[ "${listType}" == "regex_blacklist" ]]; then + param="-regex" + type="regex blacklist filter" elif [[ "${listType}" == "regex_whitelist" && "${wildcard}" == true ]]; then param="-whitewild" type="wildcard whitelist" + elif [[ "${listType}" == "regex_whitelist" ]]; then + param="-whiteregex" + type="regex whitelist filter" else param="b" type="blacklist" From 09190c1735eb7482537257cab54ea4103d8ffd18 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 5 Aug 2019 21:03:47 +0200 Subject: [PATCH 188/207] Only check once for if this is a regex list or not. Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 31822c9a..60b820f5 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -94,15 +94,19 @@ HandleOther() { } ProcessDomainList() { + local is_regexlist if [[ "${listType}" == "regex_blacklist" ]]; then # Regex black filter list listname="regex blacklist filters" + is_regexlist=true elif [[ "${listType}" == "regex_whitelist" ]]; then # Regex white filter list listname="regex whitelist filters" + is_regexlist=true else # Whitelist / Blacklist listname="${listType}" + is_regexlist=false fi for dom in "${domList[@]}"; do @@ -115,7 +119,7 @@ ProcessDomainList() { # if delmode then remove from desired list but do not add to the other if ${addmode}; then AddDomain "${dom}" "${listType}" - if [[ ! "${listType}" == "regex_"*"list" ]]; then + if ! ${is_regexlist}; then RemoveDomain "${dom}" "${listAlt}" fi else From 06860ed5b49a4554026ca4f67aa1dc8f9eed06c7 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 5 Aug 2019 21:07:39 +0200 Subject: [PATCH 189/207] Group tables have only two columns. Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index b2533eb0..e7e6b791 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1082,21 +1082,21 @@ show_groups() { show_adlists() { show_db_entries "Adlists" "SELECT * FROM adlist" "4 100 7 10 13 50" - show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 100 7 10 13 50" + show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 4" } show_whitelist() { show_db_entries "Exact whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50" show_db_entries "Regex whitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50" - show_db_entries "Exact whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 100 7 10 13 50" - show_db_entries "Regex whitelist groups" "SELECT * FROM whitelist_by_group" "4 100 7 10 13 50" + show_db_entries "Exact whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 4" + show_db_entries "Regex whitelist groups" "SELECT * FROM whitelist_by_group" "4 4" } show_blacklist() { show_db_entries "Exact blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50" show_db_entries "Regex blacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50" - show_db_entries "Exact blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 100 7 10 13 50" - show_db_entries "Regex blacklist groups" "SELECT * FROM blacklist_by_group" "4 100 7 10 13 50" + show_db_entries "Exact blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 4" + show_db_entries "Regex blacklist groups" "SELECT * FROM blacklist_by_group" "4 4" } analyze_gravity_list() { From af754e3fc4c394276aadeec26f1068ef9ae18c2d Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 5 Aug 2019 21:08:36 +0200 Subject: [PATCH 190/207] Rearrange group tables directly next to the tables they refer to. Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index e7e6b791..dc353ff0 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1087,15 +1087,15 @@ show_adlists() { show_whitelist() { show_db_entries "Exact whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50" - show_db_entries "Regex whitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50" show_db_entries "Exact whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 4" + show_db_entries "Regex whitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50" show_db_entries "Regex whitelist groups" "SELECT * FROM whitelist_by_group" "4 4" } show_blacklist() { show_db_entries "Exact blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50" - show_db_entries "Regex blacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50" show_db_entries "Exact blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 4" + show_db_entries "Regex blacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50" show_db_entries "Regex blacklist groups" "SELECT * FROM blacklist_by_group" "4 4" } From 6e2e825a5fe3c4148347067e58309d55e4771bf8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 5 Aug 2019 21:10:52 +0200 Subject: [PATCH 191/207] Rename options "pihole --whiteregex" to "pihole --white-regex" for the sake of readability. The same applied for "whitewild" -> "white-wild" Signed-off-by: DL6ER --- advanced/Scripts/list.sh | 8 ++++---- pihole | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 60b820f5..6a606665 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -39,10 +39,10 @@ helpFunc() { param="-regex" type="regex blacklist filter" elif [[ "${listType}" == "regex_whitelist" && "${wildcard}" == true ]]; then - param="-whitewild" + param="-white-wild" type="wildcard whitelist" elif [[ "${listType}" == "regex_whitelist" ]]; then - param="-whiteregex" + param="-white-regex" type="regex whitelist filter" else param="b" @@ -230,8 +230,8 @@ for var in "$@"; do "-b" | "blacklist" ) listType="blacklist"; listAlt="whitelist";; "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;; "--regex" | "regex" ) listType="regex_blacklist";; - "--whiteregex" | "whiteregex" ) listType="regex_whitelist";; - "--whitewild" | "whitewild" ) listType="regex_whitelist"; wildcard=true;; + "--white-regex" | "white-regex" ) listType="regex_whitelist";; + "--white-wild" | "white-wild" ) listType="regex_whitelist"; wildcard=true;; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; diff --git a/pihole b/pihole index 5d373a47..dfd6eda9 100755 --- a/pihole +++ b/pihole @@ -440,8 +440,8 @@ case "${1}" in "-b" | "blacklist" ) listFunc "$@";; "--wild" | "wildcard" ) listFunc "$@";; "--regex" | "regex" ) listFunc "$@";; - "--whiteregex" | "whiteregex" ) listFunc "$@";; - "--whitewild" | "whitewild" ) listFunc "$@";; + "--white-regex" | "white-regex" ) listFunc "$@";; + "--white-wild" | "white-wild" ) listFunc "$@";; "-d" | "debug" ) debugFunc "$@";; "-f" | "flush" ) flushFunc "$@";; "-up" | "updatePihole" ) updatePiholeFunc "$@";; From 4371c9ba03e48daa2f40e1dbe31c1b3bd72d2447 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Mon, 5 Aug 2019 21:20:07 +0200 Subject: [PATCH 192/207] Ensure proper permissions are set for gravity.db after creation. Signed-off-by: DL6ER --- automated install/basic-install.sh | 3 +++ gravity.sh | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 1e87b943..7dbdc596 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1908,6 +1908,9 @@ installPihole() { chmod a+rx /var/www/html # Give pihole access to the Web server group usermod -a -G ${LIGHTTPD_GROUP} pihole + # Give lighttpd access to the pihole group so the web interface can + # manage the gravity.db database + usermod -a -G pihole ${LIGHTTPD_USER} # If the lighttpd command is executable, if is_command lighty-enable-mod ; then # enable fastcgi and fastcgi-php diff --git a/gravity.sh b/gravity.sh index f9ecc1d1..1ad43df1 100755 --- a/gravity.sh +++ b/gravity.sh @@ -85,6 +85,10 @@ fi # Generate new sqlite3 file from schema template generate_gravity_database() { sqlite3 "${gravityDBfile}" < "${gravityDBschema}" + + # Ensure proper permissions are set for the newly created database + chown pihole:pihole "${gravityDBfile}" + chmod g+w "${piholeDir}" "${gravityDBfile}" } # Import domains from file and store them in the specified database table From dc93462d42e77b488883f6a392e67a5561d303c5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 6 Aug 2019 20:28:00 +0200 Subject: [PATCH 193/207] Group table has only two columns Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index dc353ff0..38861849 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1077,7 +1077,7 @@ show_db_entries() { } show_groups() { - show_db_entries "Groups" "SELECT * FROM \"group\"" "4 100 7 10 13 50" + show_db_entries "Groups" "SELECT * FROM \"group\"" "4 4" } show_adlists() { From b2d8c4374b8d1b36518c3dbafcb3c502d9d5857d Mon Sep 17 00:00:00 2001 From: snapsl Date: Wed, 14 Aug 2019 23:28:13 +0200 Subject: [PATCH 194/207] tweaked code style of webpage.sh Signed-off-by: snapsl --- advanced/Scripts/webpage.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 9db10bda..b799d67d 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -87,9 +87,9 @@ SetTemperatureUnit() { HashPassword() { # Compute password hash twice to avoid rainbow table vulnerability - return=$(echo -n ${1} | sha256sum | sed 's/\s.*$//') - return=$(echo -n ${return} | sha256sum | sed 's/\s.*$//') - echo ${return} + return=$(echo -n "${1}" | sha256sum | sed 's/\s.*$//') + return=$(echo -n "${return}" | sha256sum | sed 's/\s.*$//') + echo "${return}" } SetWebPassword() { @@ -143,18 +143,18 @@ ProcessDNSSettings() { delete_dnsmasq_setting "server" COUNTER=1 - while [[ 1 ]]; do + while true ; do var=PIHOLE_DNS_${COUNTER} if [ -z "${!var}" ]; then break; fi add_dnsmasq_setting "server" "${!var}" - let COUNTER=COUNTER+1 + (( COUNTER++ )) done # The option LOCAL_DNS_PORT is deprecated # We apply it once more, and then convert it into the current format - if [ ! -z "${LOCAL_DNS_PORT}" ]; then + if [ -n "${LOCAL_DNS_PORT}" ]; then add_dnsmasq_setting "server" "127.0.0.1#${LOCAL_DNS_PORT}" add_setting "PIHOLE_DNS_${COUNTER}" "127.0.0.1#${LOCAL_DNS_PORT}" delete_setting "LOCAL_DNS_PORT" @@ -184,7 +184,7 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 delete_dnsmasq_setting "host-record" - if [ ! -z "${HOSTRECORD}" ]; then + if [ -n "${HOSTRECORD}" ]; then add_dnsmasq_setting "host-record" "${HOSTRECORD}" fi @@ -538,7 +538,8 @@ Interfaces: } Teleporter() { - local datetimestamp=$(date "+%Y-%m-%d_%H-%M-%S") + datetimestamp=$(date "+%Y-%m-%d_%H-%M-%S") + local datetimestamp php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-teleporter_${datetimestamp}.tar.gz" } From 20a839fef58e463333ba1c08d5fe9d9bee8fd5e1 Mon Sep 17 00:00:00 2001 From: snapsl Date: Thu, 15 Aug 2019 11:20:55 +0200 Subject: [PATCH 195/207] fixed local declaration before assignment Signed-off-by: snapsl --- advanced/Scripts/webpage.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index b799d67d..39299960 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -538,8 +538,8 @@ Interfaces: } Teleporter() { - datetimestamp=$(date "+%Y-%m-%d_%H-%M-%S") local datetimestamp + datetimestamp=$(date "+%Y-%m-%d_%H-%M-%S") php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-teleporter_${datetimestamp}.tar.gz" } From 3e78ed95d4cc6a7ef50d0bca11fcc17edb8eb3c5 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 17 Aug 2019 15:04:04 +0200 Subject: [PATCH 196/207] Fix displaying options for table "group" in the debugger. Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 38861849..134b15ce 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1077,7 +1077,7 @@ show_db_entries() { } show_groups() { - show_db_entries "Groups" "SELECT * FROM \"group\"" "4 4" + show_db_entries "Groups" "SELECT * FROM \"group\"" "4 4 30 50" } show_adlists() { From b1838512b25bf3715d629c5f43fc27539f70d4ac Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 22 Aug 2019 13:39:58 +0200 Subject: [PATCH 197/207] Explicitly select columns (and their order) when listing the databaes tables. Print timestamps translated to strings instead of printing the integer timestamps. Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 134b15ce..7ba03a37 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1081,22 +1081,22 @@ show_groups() { } show_adlists() { - show_db_entries "Adlists" "SELECT * FROM adlist" "4 100 7 10 13 50" + show_db_entries "Adlists" "SELECT id,address,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM adlist" "4 100 7 19 19 50" show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 4" } show_whitelist() { - show_db_entries "Exact whitelist" "SELECT * FROM whitelist" "4 100 7 10 13 50" - show_db_entries "Exact whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 4" - show_db_entries "Regex whitelist" "SELECT * FROM regex_whitelist" "4 100 7 10 13 50" - show_db_entries "Regex whitelist groups" "SELECT * FROM whitelist_by_group" "4 4" + show_db_entries "Exact whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM whitelist" "4 100 7 19 19 50" + show_db_entries "Exact whitelist groups" "SELECT * FROM whitelist_by_group" "4 4" + show_db_entries "Regex whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM regex_whitelist" "4 100 7 19 19 50" + show_db_entries "Regex whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 4" } show_blacklist() { - show_db_entries "Exact blacklist" "SELECT * FROM blacklist" "4 100 7 10 13 50" - show_db_entries "Exact blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 4" - show_db_entries "Regex blacklist" "SELECT * FROM regex_blacklist" "4 100 7 10 13 50" - show_db_entries "Regex blacklist groups" "SELECT * FROM blacklist_by_group" "4 4" + show_db_entries "Exact blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM blacklist" "4 100 7 19 19 50" + show_db_entries "Exact blacklist groups" "SELECT * FROM blacklist_by_group" "4 4" + show_db_entries "Regex blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM regex_blacklist" "4 100 7 19 19 50" + show_db_entries "Regex blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 4" } analyze_gravity_list() { From cc40c18f49ccf9dcf926cd18bb9dc1880345be3c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 22 Aug 2019 13:54:46 +0200 Subject: [PATCH 198/207] Wrap upgrade script commands in a transaction. Signed-off-by: DL6ER --- advanced/Scripts/database_migration/gravity/1_to_2.sql | 6 ++++++ advanced/Scripts/database_migration/gravity/2_to_3.sql | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql index 90a48418..6d57a6fe 100644 --- a/advanced/Scripts/database_migration/gravity/1_to_2.sql +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -1,3 +1,7 @@ +.timeout 30000 + +BEGIN TRANSACTION; + CREATE TABLE domain_audit ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -6,3 +10,5 @@ CREATE TABLE domain_audit ); UPDATE info SET value = 2 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql index e121f28c..fd7c24d2 100644 --- a/advanced/Scripts/database_migration/gravity/2_to_3.sql +++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql @@ -2,6 +2,8 @@ PRAGMA FOREIGN_KEYS=OFF; +BEGIN TRANSACTION; + ALTER TABLE regex RENAME TO regex_blacklist; CREATE TABLE regex_blacklist_by_group @@ -59,3 +61,5 @@ CREATE TRIGGER tr_regex_whitelist_update AFTER UPDATE ON regex_whitelist UPDATE info SET value = 3 WHERE property = 'version'; + +COMMIT; From aef7892de68a272df0dd62bd18e28a8790af66fa Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 22 Aug 2019 13:57:01 +0200 Subject: [PATCH 199/207] Add missing hyphens. Signed-off-by: DL6ER --- manpages/pihole.8 | 4 ++-- pihole | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 11923392..ed012092 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -71,7 +71,7 @@ Available commands and options: Add or removes specified regex filter to the regex blacklist .br -\fB--whiteregex\fR [options] [ ] +\fB--white-regex\fR [options] [ ] .br Add or removes specified regex filter to the regex whitelist .br @@ -81,7 +81,7 @@ Available commands and options: Add or removes specified domain to the wildcard blacklist .br -\fB--whitewild\fR [options] [ ] +\fB--white-wild\fR [options] [ ] .br Add or removes specified domain to the wildcard whitelist .br diff --git a/pihole b/pihole index dfd6eda9..1d9f0809 100755 --- a/pihole +++ b/pihole @@ -375,10 +375,10 @@ Add '-h' after specific commands for more information on usage Whitelist/Blacklist Options: -w, whitelist Whitelist domain(s) -b, blacklist Blacklist domain(s) - --regex, regex Regex blacklist domains(s) - --whiteregex Regex whitelist domains(s) - --wild, wildcard Wildcard blacklist domain(s) - --whitewild Wildcard whitelist domain(s) + --regex, regex Regex blacklist domains(s) + --white-regex Regex whitelist domains(s) + --wild, wildcard Wildcard blacklist domain(s) + --white-wild Wildcard whitelist domain(s) Add '-h' for more info on whitelist/blacklist usage Debugging Options: From 42ccc1ef24cf80d53d0b20e9e01d0a6d1a66a992 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 22 Aug 2019 14:06:42 +0200 Subject: [PATCH 200/207] Add support for regex whitelist in "pihole -q". Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index a587f238..4bffe251 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -128,25 +128,24 @@ scanDatabaseTable() { done } -# Scan Whitelist and Blacklist -scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" -scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" - -# Scan Regex table -mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_blacklist" 2> /dev/null) +scanRegexDatabaseTable() { +local domain list +domain="${1}" +list="${2}" +mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_${list}" 2> /dev/null) # If we have regexps to process if [[ "${#regexList[@]}" -ne 0 ]]; then # Split regexps over a new line str_regexList=$(printf '%s\n' "${regexList[@]}") - # Check domainQuery against regexps - mapfile -t regexMatches < <(scanList "${domainQuery}" "${str_regexList}" "regex") + # Check domain against regexps + mapfile -t regexMatches < <(scanList "${domain}" "${str_regexList}" "regex") # If there were regex matches if [[ "${#regexMatches[@]}" -ne 0 ]]; then # Split matching regexps over a new line str_regexMatches=$(printf '%s\n' "${regexMatches[@]}") # Form a "matched" message - str_message="${matchType^} found in ${COL_BOLD}Regex list${COL_NC}" + str_message="${matchType^} found in ${COL_BOLD}Regex ${list}${COL_NC}" # Form a "results" message str_result="${COL_BOLD}${str_regexMatches}${COL_NC}" # If we are displaying more than just the source of the block @@ -159,11 +158,20 @@ if [[ "${#regexList[@]}" -ne 0 ]]; then # shellcheck disable=SC2001 echo "${str_result}" | sed 's/^/ /' else - echo "π Regex list" + echo "π Regex ${list}" exit 0 fi fi fi +} + +# Scan Whitelist and Blacklist +scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" +scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" + +# Scan Regex table +scanRegexDatabaseTable "${domainQuery}" "whitelist" +scanRegexDatabaseTable "${domainQuery}" "blacklist" # Get version sorted *.domains filenames (without dir path) lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)") From 23b688287f4fdb9c809053880594f894c4cfbfc8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Thu, 22 Aug 2019 14:12:58 +0200 Subject: [PATCH 201/207] Fix indentation in query.sh. No functional change in this commit. Signed-off-by: DL6ER --- advanced/Scripts/query.sh | 88 ++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 43 deletions(-) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 4bffe251..035adaac 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -36,14 +36,14 @@ scanList(){ # /dev/null forces filename to be printed when only one list has been generated # shellcheck disable=SC2086 case "${type}" in - "exact" ) grep -i -E -l "(^|(?/dev/null;; - # Create array of regexps - # Iterate through each regexp and check whether it matches the domainQuery - # If it does, print the matching regexp and continue looping - # Input 1 - regexps | Input 2 - domainQuery - "regex" ) awk 'NR==FNR{regexps[$0];next}{for (r in regexps)if($0 ~ r)print r}' \ - <(echo "${lists}") <(echo "${domain}") 2>/dev/null;; - * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; + "exact" ) grep -i -E -l "(^|(?/dev/null;; + # Create array of regexps + # Iterate through each regexp and check whether it matches the domainQuery + # If it does, print the matching regexp and continue looping + # Input 1 - regexps | Input 2 - domainQuery + "regex" ) awk 'NR==FNR{regexps[$0];next}{for (r in regexps)if($0 ~ r)print r}' \ + <(echo "${lists}") <(echo "${domain}") 2>/dev/null;; + * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; esac } @@ -100,8 +100,8 @@ scanDatabaseTable() { # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. case "${type}" in - "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; - * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; + * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; esac # Send prepared query to gravity database @@ -129,40 +129,42 @@ scanDatabaseTable() { } scanRegexDatabaseTable() { -local domain list -domain="${1}" -list="${2}" -mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_${list}" 2> /dev/null) + local domain list + domain="${1}" + list="${2}" -# If we have regexps to process -if [[ "${#regexList[@]}" -ne 0 ]]; then - # Split regexps over a new line - str_regexList=$(printf '%s\n' "${regexList[@]}") - # Check domain against regexps - mapfile -t regexMatches < <(scanList "${domain}" "${str_regexList}" "regex") - # If there were regex matches - if [[ "${#regexMatches[@]}" -ne 0 ]]; then - # Split matching regexps over a new line - str_regexMatches=$(printf '%s\n' "${regexMatches[@]}") - # Form a "matched" message - str_message="${matchType^} found in ${COL_BOLD}Regex ${list}${COL_NC}" - # Form a "results" message - str_result="${COL_BOLD}${str_regexMatches}${COL_NC}" - # If we are displaying more than just the source of the block - if [[ -z "${blockpage}" ]]; then - # Set the wildcard match flag - wcMatch=true - # Echo the "matched" message, indented by one space - echo " ${str_message}" - # Echo the "results" message, each line indented by three spaces - # shellcheck disable=SC2001 - echo "${str_result}" | sed 's/^/ /' - else - echo "π Regex ${list}" - exit 0 - fi - fi -fi + # Query all regex from the corresponding database tables + mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_${list}" 2> /dev/null) + + # If we have regexps to process + if [[ "${#regexList[@]}" -ne 0 ]]; then + # Split regexps over a new line + str_regexList=$(printf '%s\n' "${regexList[@]}") + # Check domain against regexps + mapfile -t regexMatches < <(scanList "${domain}" "${str_regexList}" "regex") + # If there were regex matches + if [[ "${#regexMatches[@]}" -ne 0 ]]; then + # Split matching regexps over a new line + str_regexMatches=$(printf '%s\n' "${regexMatches[@]}") + # Form a "matched" message + str_message="${matchType^} found in ${COL_BOLD}Regex ${list}${COL_NC}" + # Form a "results" message + str_result="${COL_BOLD}${str_regexMatches}${COL_NC}" + # If we are displaying more than just the source of the block + if [[ -z "${blockpage}" ]]; then + # Set the wildcard match flag + wcMatch=true + # Echo the "matched" message, indented by one space + echo " ${str_message}" + # Echo the "results" message, each line indented by three spaces + # shellcheck disable=SC2001 + echo "${str_result}" | sed 's/^/ /' + else + echo "π Regex ${list}" + exit 0 + fi + fi + fi } # Scan Whitelist and Blacklist From 6faddfcd3d0a0ae00c0de3833b43013c60efec96 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Fri, 23 Aug 2019 10:09:52 +0200 Subject: [PATCH 202/207] Print timestamps in local time zone of the Pi-hole. Signed-off-by: DL6ER --- advanced/Scripts/piholeDebug.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index c9280f45..84e34416 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1109,21 +1109,21 @@ show_groups() { } show_adlists() { - show_db_entries "Adlists" "SELECT id,address,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM adlist" "4 100 7 19 19 50" + show_db_entries "Adlists" "SELECT id,address,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM adlist" "4 100 7 19 19 50" show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 4" } show_whitelist() { - show_db_entries "Exact whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM whitelist" "4 100 7 19 19 50" + show_db_entries "Exact whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM whitelist" "4 100 7 19 19 50" show_db_entries "Exact whitelist groups" "SELECT * FROM whitelist_by_group" "4 4" - show_db_entries "Regex whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM regex_whitelist" "4 100 7 19 19 50" + show_db_entries "Regex whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM regex_whitelist" "4 100 7 19 19 50" show_db_entries "Regex whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 4" } show_blacklist() { - show_db_entries "Exact blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM blacklist" "4 100 7 19 19 50" + show_db_entries "Exact blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM blacklist" "4 100 7 19 19 50" show_db_entries "Exact blacklist groups" "SELECT * FROM blacklist_by_group" "4 4" - show_db_entries "Regex blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch') date_added,datetime(date_modified,'unixepoch') date_modified,comment FROM regex_blacklist" "4 100 7 19 19 50" + show_db_entries "Regex blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM regex_blacklist" "4 100 7 19 19 50" show_db_entries "Regex blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 4" } From ca8982494ba37002377dd0bf78b023381a61bfce Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sun, 1 Sep 2019 14:42:07 +0200 Subject: [PATCH 203/207] Store timestamp when the gravity table was last updated successfully. This fixes https://github.com/pi-hole/AdminLTE/issues/989 Signed-off-by: DL6ER --- gravity.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/gravity.sh b/gravity.sh index 2fc26d49..98747f35 100755 --- a/gravity.sh +++ b/gravity.sh @@ -90,6 +90,16 @@ generate_gravity_database() { chmod g+w "${piholeDir}" "${gravityDBfile}" } +update_gravity_timestamp() { + # Update timestamp when the gravity table was last updated successfully + output=$( { sqlite3 "${gravityDBfile}" <<< "INSERT OR REPLACE INTO info (property,value) values (\"updated\",cast(strftime('%s', 'now') as int));"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to update gravity timestamp in database ${gravityDBfile}\\n ${output}" + fi +} + # Import domains from file and store them in the specified database table database_table_from_file() { # Define locals @@ -749,6 +759,8 @@ fi gravity_generateLocalList gravity_ShowCount +update_gravity_timestamp + gravity_Cleanup echo "" From 525ec8cd01cfc069f4c670ac6e9c0aef0a17c02c Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 7 Sep 2019 08:44:03 +0200 Subject: [PATCH 204/207] Signal to Firefox that the local network is unsuitable for DNS-over-HTTPS Signed-off-by: DL6ER --- advanced/01-pihole.conf | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf index cd74e186..8aa35fe1 100644 --- a/advanced/01-pihole.conf +++ b/advanced/01-pihole.conf @@ -41,3 +41,8 @@ log-facility=/var/log/pihole.log local-ttl=2 log-async + +# Signal to Firefox that the local network is unsuitable for DNS-over-HTTPS +# This follows https://support.mozilla.org/en-US/kb/configuring-networks-disable-dns-over-https +# (sourced 7th September 2019) +server=/use-application-dns.net/ From 1f36ec48e3af54d849a191f48bdb6559158ce5f8 Mon Sep 17 00:00:00 2001 From: DL6ER Date: Sat, 7 Sep 2019 23:11:20 +0200 Subject: [PATCH 205/207] Add use-application-dns.net = NXDOMAIN in ProcessDNSSettings rather than in the template so we can ensure that it will survive config-renewals. Signed-off-by: DL6ER --- advanced/01-pihole.conf | 5 ----- advanced/Scripts/webpage.sh | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf index 8aa35fe1..cd74e186 100644 --- a/advanced/01-pihole.conf +++ b/advanced/01-pihole.conf @@ -41,8 +41,3 @@ log-facility=/var/log/pihole.log local-ttl=2 log-async - -# Signal to Firefox that the local network is unsuitable for DNS-over-HTTPS -# This follows https://support.mozilla.org/en-US/kb/configuring-networks-disable-dns-over-https -# (sourced 7th September 2019) -server=/use-application-dns.net/ diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 39299960..356c20ef 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -212,6 +212,11 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_DOMAIN}/${CONDITIONAL_FORWARDING_IP}" add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_REVERSE}/${CONDITIONAL_FORWARDING_IP}" fi + + # Prevent Firefox from automatically switching over to DNS-over-HTTPS + # This follows https://support.mozilla.org/en-US/kb/configuring-networks-disable-dns-over-https + # (sourced 7th September 2019) + add_dnsmasq_setting "server=/use-application-dns.net/" } SetDNSServers() { From f582344b9ac6698a5e500b20739994a6fed27cab Mon Sep 17 00:00:00 2001 From: DL6ER Date: Tue, 17 Sep 2019 21:59:48 +0200 Subject: [PATCH 206/207] "No default index.lighttpd.html file found... not backing up" is not an error. Signed-off-by: DL6ER --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 25c66ab7..e9684254 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1692,7 +1692,7 @@ installPiholeWeb() { # Otherwise, else # don't do anything - printf "%b %b %s\\n" "${OVER}" "${CROSS}" "${str}" + printf "%b %b %s\\n" "${OVER}" "${INFO}" "${str}" printf " No default index.lighttpd.html file found... not backing up\\n" fi From 9a6deb5a1a4c1d811c5a906d107e06d6973e7fcb Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 17 Sep 2019 21:16:49 +0100 Subject: [PATCH 207/207] Fix tests Signed-off-by: Adam Warner --- test/test_automated_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index cce11857..e8a4dede 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -338,7 +338,7 @@ def test_installPiholeWeb_fresh_install_no_errors(Pihole): expected_stdout = tick_box + (' Creating directory for blocking page, ' 'and copying files') assert expected_stdout in installWeb.stdout - expected_stdout = cross_box + ' Backing up index.lighttpd.html' + expected_stdout = info_box + ' Backing up index.lighttpd.html' assert expected_stdout in installWeb.stdout expected_stdout = ('No default index.lighttpd.html file found... ' 'not backing up')