diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf index 502293bf..40a117fe 100644 --- a/advanced/01-pihole.conf +++ b/advanced/01-pihole.conf @@ -18,8 +18,6 @@ # WITHIN /etc/dnsmasq.d/yourname.conf # ############################################################################### -addn-hosts=/etc/pihole/gravity.list -addn-hosts=/etc/pihole/black.list addn-hosts=/etc/pihole/local.list domain-needed diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index b4fffbaf..fa81348b 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -11,46 +11,45 @@ # Globals basename=pihole piholeDir=/etc/"${basename}" -whitelist="${piholeDir}"/whitelist.txt -blacklist="${piholeDir}"/blacklist.txt +gravityDBfile="${piholeDir}/gravity.db" -readonly regexlist="/etc/pihole/regex.list" reload=false addmode=true verbose=true wildcard=false +web=false domList=() -listMain="" -listAlt="" +listType="" +listname="" colfile="/opt/pihole/COL_TABLE" source ${colfile} helpFunc() { - if [[ "${listMain}" == "${whitelist}" ]]; then + if [[ "${listType}" == "whitelist" ]]; then param="w" - type="white" - elif [[ "${listMain}" == "${regexlist}" && "${wildcard}" == true ]]; then + type="whitelist" + elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then param="-wild" - type="wildcard black" - elif [[ "${listMain}" == "${regexlist}" ]]; then + type="wildcard blacklist" + elif [[ "${listType}" == "regex" ]]; then param="-regex" - type="regex black" + type="regex filter" else param="b" - type="black" + type="blacklist" fi echo "Usage: pihole -${param} [options] Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com' -${type^}list one or more domains +${type^} one or more domains Options: - -d, --delmode Remove domain(s) from the ${type}list - -nr, --noreload Update ${type}list without refreshing dnsmasq + -d, --delmode Remove domain(s) from the ${type} + -nr, --noreload Update ${type} without reloading the DNS server -q, --quiet Make output less verbose -h, --help Show this help dialog -l, --list Display all your ${type}listed domains @@ -73,7 +72,7 @@ HandleOther() { # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then - if [[ "${listMain}" == "${regexlist}" && "${wildcard}" == false ]]; then + if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then validDomain="${domain}" else validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check @@ -88,178 +87,143 @@ HandleOther() { fi } -PoplistFile() { - # Check whitelist file exists, and if not, create it - if [[ ! -f "${whitelist}" ]]; then - touch "${whitelist}" - chmod 644 "${whitelist}" - fi - - # Check blacklist file exists, and if not, create it - if [[ ! -f "${blacklist}" ]]; then - touch "${blacklist}" - chmod 644 "${blacklist}" +ProcessDomainList() { + if [[ "${listType}" == "regex" ]]; then + # Regex filter list + listname="regex filters" + else + # Whitelist / Blacklist + listname="${listType}" fi for dom in "${domList[@]}"; do - # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other + # Format domain into regex filter if requested + if [[ "${wildcard}" == true ]]; then + dom="(^|\\.)${dom//\./\\.}$" + fi + + # Logic: If addmode then add to desired list and remove from the other; + # if delmode then remove from desired list but do not add to the other if ${addmode}; then - AddDomain "${dom}" "${listMain}" - RemoveDomain "${dom}" "${listAlt}" + AddDomain "${dom}" "${listType}" + if [[ ! "${listType}" == "regex" ]]; then + RemoveDomain "${dom}" "${listAlt}" + fi else - RemoveDomain "${dom}" "${listMain}" + RemoveDomain "${dom}" "${listType}" fi done } AddDomain() { + local domain list num + # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input + domain="$1" list="$2" - domain=$(EscapeRegexp "$1") - [[ "${list}" == "${whitelist}" ]] && listname="whitelist" - [[ "${list}" == "${blacklist}" ]] && listname="blacklist" + # Is the domain in the list we want to add it to? + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")" - if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then - [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only" - [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only" - bool=true - # Is the domain in the list we want to add it to? - grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false - - if [[ "${bool}" == false ]]; then - # Domain not found in the whitelist file, add it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${1} to ${listname}..." - fi - reload=true - # Add it to the list we want to add it to - echo "$1" >> "${list}" - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" - fi - fi - elif [[ "${list}" == "${regexlist}" ]]; then - [[ -z "${type}" ]] && type="--wildcard-only" - bool=true - domain="${1}" - - [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$" - - # Is the domain in the list? - # Search only for exactly matching lines - grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false - - if [[ "${bool}" == false ]]; then - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${domain} to regex list..." - fi - reload="restart" - echo "$domain" >> "${regexlist}" - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${domain} already exists in regex list, no need to add!" - fi - fi + if [[ "${num}" -ne 0 ]]; then + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" + fi + return fi + + # Domain not found in the table, add it! + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} Adding ${1} to the ${listname}..." + fi + reload=true + # Insert only the domain here. The enabled and date_added fields will be filled + # with their default values (enabled = true, date_added = current timestamp) + sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES ('${domain}');" } RemoveDomain() { + local domain list num + # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input + domain="$1" list="$2" - domain=$(EscapeRegexp "$1") - [[ "${list}" == "${whitelist}" ]] && listname="whitelist" - [[ "${list}" == "${blacklist}" ]] && listname="blacklist" + # Is the domain in the list we want to remove it from? + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")" - if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then - bool=true - [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only" - [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only" - # Is it in the list? Logic follows that if its whitelisted it should not be blacklisted and vice versa - grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false - if [[ "${bool}" == true ]]; then - # Remove it from the other one - echo -e " ${INFO} Removing $1 from ${listname}..." - # /I flag: search case-insensitive - sed -i "/${domain}/Id" "${list}" - reload=true - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" - fi - fi - elif [[ "${list}" == "${regexlist}" ]]; then - [[ -z "${type}" ]] && type="--wildcard-only" - domain="${1}" - - [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$" - - bool=true - # Is it in the list? - grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false - if [[ "${bool}" == true ]]; then - # Remove it from the other one - echo -e " ${INFO} Removing $domain from regex list..." - local lineNumber - lineNumber=$(grep -Fnx "$domain" "${list}" | cut -f1 -d:) - sed -i "${lineNumber}d" "${list}" - reload=true - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${domain} does not exist in regex list, no need to remove!" - fi - fi + if [[ "${num}" -eq 0 ]]; then + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} does not exist in ${list}, no need to remove!" + fi + return fi -} -# Update Gravity -Reload() { - echo "" - pihole -g --skip-download "${type:-}" + # Domain found in the table, remove it! + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} Removing ${1} from the ${listname}..." + fi + reload=true + # Remove it from the current list + sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = '${domain}';" } Displaylist() { - if [[ -f ${listMain} ]]; then - if [[ "${listMain}" == "${whitelist}" ]]; then - string="gravity resistant domains" - else - string="domains caught in the sinkhole" - fi - verbose=false - echo -e "Displaying $string:\n" - count=1 - while IFS= read -r RD || [ -n "${RD}" ]; do - echo " ${count}: ${RD}" - count=$((count+1)) - done < "${listMain}" + local list listname count num_pipes domain enabled status nicedate + + listname="${listType}" + data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)" + + if [[ -z $data ]]; then + echo -e "Not showing empty ${listname}" else - echo -e " ${COL_LIGHT_RED}${listMain} does not exist!${COL_NC}" + echo -e "Displaying ${listname}:" + count=1 + while IFS= read -r line + do + # Count number of pipes seen in this line + # This is necessary because we can only detect the pipe separating the fields + # from the end backwards as the domain (which is the first field) may contain + # pipe symbols as they are perfectly valid regex filter control characters + num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")" + + # Extract domain and enabled status based on the obtained number of pipe characters + domain="$(cut -d'|' -f"-$((num_pipes-1))" <<< "${line}")" + enabled="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")" + datemod="$(cut -d'|' -f"$((num_pipes+1))" <<< "${line}")" + + # Translate boolean status into human readable string + if [[ "${enabled}" -eq 1 ]]; then + status="enabled" + else + status="disabled" + fi + + # Get nice representation of numerical date stored in database + nicedate=$(date --rfc-2822 -d "@${datemod}") + + echo " ${count}: ${domain} (${status}, last modified ${nicedate})" + count=$((count+1)) + done <<< "${data}" fi exit 0; } NukeList() { - if [[ -f "${listMain}" ]]; then - # Back up original list - cp -p "${listMain}" "${listMain}.bck~" - # Empty out file - echo "" > "${listMain}" - chmod 644 "${listMain}" - fi + sqlite3 "${gravityDBfile}" "DELETE FROM ${listType};" } for var in "$@"; do case "${var}" in - "-w" | "whitelist" ) listMain="${whitelist}"; listAlt="${blacklist}";; - "-b" | "blacklist" ) listMain="${blacklist}"; listAlt="${whitelist}";; - "--wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;; - "--regex" | "regex" ) listMain="${regexlist}";; + "-w" | "whitelist" ) listType="whitelist"; listAlt="blacklist";; + "-b" | "blacklist" ) listType="blacklist"; listAlt="whitelist";; + "--wild" | "wildcard" ) listType="regex"; wildcard=true;; + "--regex" | "regex" ) listType="regex";; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; "-h" | "--help" ) helpFunc;; "-l" | "--list" ) Displaylist;; "--nuke" ) NukeList;; + "--web" ) web=true;; * ) HandleOther "${var}";; esac done @@ -270,9 +234,13 @@ if [[ $# = 0 ]]; then helpFunc fi -PoplistFile +ProcessDomainList + +# Used on web interface +if $web; then +echo "DONE" +fi if [[ "${reload}" != false ]]; then - # Ensure that "restart" is used for Wildcard updates - Reload "${reload}" + pihole restartdns reload fi diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh old mode 100644 new mode 100755 index 69a3c7a4..a4ac895b --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -11,7 +11,7 @@ # Globals piholeDir="/etc/pihole" -adListsList="$piholeDir/adlists.list" +gravityDBfile="${piholeDir}/gravity.db" wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf" options="$*" adlist="" @@ -73,11 +73,6 @@ Options: exit 0 fi -if [[ ! -e "$adListsList" ]]; then - echo -e "${COL_LIGHT_RED}The file $adListsList was not found${COL_NC}" - exit 1 -fi - # Handle valid options if [[ "${options}" == *"-bp"* ]]; then exact="exact"; blockpage=true @@ -186,11 +181,8 @@ fi # Get adlist file content as array if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then - for adlistUrl in $(< "${adListsList}"); do - if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then - adlists+=("${adlistUrl}") - fi - done + # Retrieve source URLs from gravity database + mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" fi # Print "Exact matches for" title diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 467eebd6..abac22d4 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -17,6 +17,8 @@ readonly FTLconf="/etc/pihole/pihole-FTL.conf" # 03 -> wildcards readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf" +readonly gravityDBfile="/etc/pihole/gravity.db" + coltable="/opt/pihole/COL_TABLE" if [[ -f ${coltable} ]]; then source ${coltable} @@ -386,19 +388,17 @@ SetWebUILayout() { } CustomizeAdLists() { - list="/etc/pihole/adlists.list" + local address + address="${args[3]}" if [[ "${args[2]}" == "enable" ]]; then - sed -i "\\@${args[3]}@s/^#http/http/g" "${list}" + sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 1 WHERE address = '${address}'" elif [[ "${args[2]}" == "disable" ]]; then - sed -i "\\@${args[3]}@s/^http/#http/g" "${list}" + sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 0 WHERE address = '${address}'" elif [[ "${args[2]}" == "add" ]]; then - if [[ $(grep -c "^${args[3]}$" "${list}") -eq 0 ]] ; then - echo "${args[3]}" >> ${list} - fi + sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlists (address) VALUES ('${address}')" elif [[ "${args[2]}" == "del" ]]; then - var=$(echo "${args[3]}" | sed 's/\//\\\//g') - sed -i "/${var}/Id" "${list}" + sqlite3 "${gravityDBfile}" "DELETE FROM adlists WHERE address = '${address}'" else echo "Not permitted" return 1 diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql new file mode 100644 index 00000000..372a4a29 --- /dev/null +++ b/advanced/Templates/gravity.db.sql @@ -0,0 +1,92 @@ +CREATE TABLE whitelist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TABLE blacklist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TABLE regex +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TABLE adlists +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + address TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); +CREATE TABLE gravity +( + domain TEXT PRIMARY KEY +); +CREATE TABLE info +( + property TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +INSERT INTO info VALUES("version","1"); + +CREATE VIEW vw_gravity AS SELECT a.domain + FROM gravity a + WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1); + +CREATE VIEW vw_whitelist AS SELECT a.domain + FROM whitelist a + WHERE a.enabled == 1 + ORDER BY a.id; + +CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist + BEGIN + UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + +CREATE VIEW vw_blacklist AS SELECT a.domain + FROM blacklist a + WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist + ORDER BY a.id; + +CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist + BEGIN + UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + +CREATE VIEW vw_regex AS SELECT a.domain + FROM regex a + WHERE a.enabled == 1 + ORDER BY a.id; + +CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex + BEGIN + UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + +CREATE VIEW vw_adlists AS SELECT a.address + FROM adlists a + WHERE a.enabled == 1 + ORDER BY a.id; + +CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists + BEGIN + UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; + END; + diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 9520c44c..18c47485 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1234,6 +1234,7 @@ version_check_dnsmasq() { local dnsmasq_conf="/etc/dnsmasq.conf" local dnsmasq_conf_orig="/etc/dnsmasq.conf.orig" local dnsmasq_pihole_id_string="addn-hosts=/etc/pihole/gravity.list" + local dnsmasq_pihole_id_string2="# Dnsmasq config for Pi-hole's FTLDNS" local dnsmasq_original_config="${PI_HOLE_LOCAL_REPO}/advanced/dnsmasq.conf.original" local dnsmasq_pihole_01_snippet="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf" local dnsmasq_pihole_01_location="/etc/dnsmasq.d/01-pihole.conf" @@ -1241,8 +1242,9 @@ version_check_dnsmasq() { # If the dnsmasq config file exists if [[ -f "${dnsmasq_conf}" ]]; then printf " %b Existing dnsmasq.conf found..." "${INFO}" - # If gravity.list is found within this file, we presume it's from older versions on Pi-hole, - if grep -q ${dnsmasq_pihole_id_string} ${dnsmasq_conf}; then + # If a specific string is found within this file, we presume it's from older versions on Pi-hole, + if grep -q "${dnsmasq_pihole_id_string}" "${dnsmasq_conf}" || + grep -q "${dnsmasq_pihole_id_string2}" "${dnsmasq_conf}"; then printf " it is from a previous Pi-hole install.\\n" printf " %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}" # so backup the original file diff --git a/gravity.sh b/gravity.sh index 2a22b792..c01feb9a 100755 --- a/gravity.sh +++ b/gravity.sh @@ -23,31 +23,27 @@ PIHOLE_COMMAND="/usr/local/bin/${basename}" piholeDir="/etc/${basename}" -adListFile="${piholeDir}/adlists.list" -adListDefault="${piholeDir}/adlists.default" - +# Legacy (pre v5.0) list file locations whitelistFile="${piholeDir}/whitelist.txt" blacklistFile="${piholeDir}/blacklist.txt" regexFile="${piholeDir}/regex.list" +adListFile="${piholeDir}/adlists.list" -adList="${piholeDir}/gravity.list" -blackList="${piholeDir}/black.list" localList="${piholeDir}/local.list" VPNList="/etc/openvpn/ipp.txt" +piholeGitDir="/etc/.pihole" +gravityDBfile="${piholeDir}/gravity.db" +gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql" +optimize_database=false + domainsExtension="domains" matterAndLight="${basename}.0.matterandlight.txt" parsedMatter="${basename}.1.parsedmatter.txt" -whitelistMatter="${basename}.2.whitelistmatter.txt" -accretionDisc="${basename}.3.accretionDisc.txt" preEventHorizon="list.preEventHorizon" -skipDownload="false" - resolver="pihole-FTL" -haveSourceUrls=true - # Source setupVars from install script setupVars="${piholeDir}/setupVars.conf" if [[ -f "${setupVars}" ]];then @@ -83,17 +79,104 @@ if [[ -r "${piholeDir}/pihole.conf" ]]; then echo -e " ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}" fi -# Determine if Pi-hole blocking is disabled -# If this is the case, we want to update -# gravity.list.bck and black.list.bck instead of -# gravity.list and black.list -detect_pihole_blocking_status() { - if [[ "${BLOCKING_ENABLED}" == false ]]; then - echo -e " ${INFO} Pi-hole blocking is disabled" - adList="${adList}.bck" - blackList="${blackList}.bck" +# Generate new sqlite3 file from schema template +generate_gravity_database() { + sqlite3 "${gravityDBfile}" < "${gravityDBschema}" +} + +# Import domains from file and store them in the specified database table +database_table_from_file() { + # Define locals + local table source backup_path backup_file + table="${1}" + source="${2}" + backup_path="${piholeDir}/migration_backup" + backup_file="${backup_path}/$(basename "${2}")" + + # Truncate table + output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to truncate ${table} database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi + + local tmpFile + tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")" + local timestamp + timestamp="$(date --utc +'%s')" + local inputfile + if [[ "${table}" == "gravity" ]]; then + # No need to modify the input data for the gravity table + inputfile="${source}" else - echo -e " ${INFO} Pi-hole blocking is enabled" + # Apply format for white-, blacklist, regex, and adlists tables + local rowid + declare -i rowid + rowid=1 + # Read file line by line + grep -v '^ *#' < "${source}" | while IFS= read -r domain + do + # Only add non-empty lines + if [[ ! -z "${domain}" ]]; then + echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}" + rowid+=1 + fi + done + inputfile="${tmpFile}" + fi + # Store domains in database table specified by ${table} + # Use printf as .mode and .import need to be on separate lines + # see https://unix.stackexchange.com/a/445615/83260 + output=$( { printf ".mode csv\\n.import \"%s\" %s\\n" "${inputfile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to fill table ${table} in database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi + + # Delete tmpfile + rm "${tmpFile}" > /dev/null 2>&1 || \ + echo -e " ${CROSS} Unable to remove ${tmpFile}" + + # Move source file to backup directory, create directory if not existing + mkdir -p "${backup_path}" + mv "${source}" "${backup_file}" 2> /dev/null || \ + echo -e " ${CROSS} Unable to backup ${source} to ${backup_path}" +} + +# Migrate pre-v5.0 list files to database-based Pi-hole versions +migrate_to_database() { + # Create database file only if not present + if [ -e "${gravityDBfile}" ]; then + return 0 + fi + + echo -e " ${INFO} Creating new gravity database" + generate_gravity_database + + # Migrate list files to new database + if [[ -e "${adListFile}" ]]; then + # Store adlists domains in database + echo -e " ${INFO} Migrating content of ${adListFile} into new database" + database_table_from_file "adlists" "${adListFile}" + fi + if [[ -e "${blacklistFile}" ]]; then + # Store blacklisted domains in database + echo -e " ${INFO} Migrating content of ${blacklistFile} into new database" + database_table_from_file "blacklist" "${blacklistFile}" + fi + if [[ -e "${whitelistFile}" ]]; then + # Store whitelisted domains in database + echo -e " ${INFO} Migrating content of ${whitelistFile} into new database" + database_table_from_file "whitelist" "${whitelistFile}" + fi + if [[ -e "${regexFile}" ]]; then + # Store regex domains in database + echo -e " ${INFO} Migrating content of ${regexFile} into new database" + database_table_from_file "regex" "${regexFile}" fi } @@ -157,15 +240,9 @@ gravity_CheckDNSResolutionAvailable() { gravity_GetBlocklistUrls() { echo -e " ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..." - if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then - # Remove superceded $adListDefault file - rm "${adListDefault}" 2> /dev/null || \ - echo -e " ${CROSS} Unable to remove ${adListDefault}" - fi - - # Retrieve source URLs from $adListFile - # Logic: Remove comments and empty lines - mapfile -t sources <<< "$(grep -v -E "^(#|$)" "${adListFile}" 2> /dev/null)" + # Retrieve source URLs from gravity database + # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true) + mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)" # Parse source domains from $sources mapfile -t sourceDomains <<< "$( @@ -182,11 +259,12 @@ gravity_GetBlocklistUrls() { if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then echo -e "${OVER} ${TICK} ${str}" + return 0 else echo -e "${OVER} ${CROSS} ${str}" echo -e " ${INFO} No source list found, or it is empty" echo "" - haveSourceUrls=false + return 1 fi } @@ -214,11 +292,9 @@ gravity_SetDownloadOptions() { *) cmd_ext="";; esac - if [[ "${skipDownload}" == false ]]; then - echo -e " ${INFO} Target: ${domain} (${url##*/})" - gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" - echo "" - fi + echo -e " ${INFO} Target: ${domain} (${url##*/})" + gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}" + echo "" done gravity_Blackbody=true } @@ -439,9 +515,7 @@ gravity_ConsolidateDownloadedBlocklists() { local str lastLine str="Consolidating blocklists" - if [[ "${haveSourceUrls}" == true ]]; then - echo -ne " ${INFO} ${str}..." - fi + echo -ne " ${INFO} ${str}..." # Empty $matterAndLight if it already exists, otherwise, create it : > "${piholeDir}/${matterAndLight}" @@ -461,9 +535,8 @@ gravity_ConsolidateDownloadedBlocklists() { fi fi done - if [[ "${haveSourceUrls}" == true ]]; then - echo -e "${OVER} ${TICK} ${str}" - fi + echo -e "${OVER} ${TICK} ${str}" + } # Parse consolidated list into (filtered, unique) domains-only format @@ -471,69 +544,45 @@ gravity_SortAndFilterConsolidatedList() { local str num str="Extracting domains from blocklists" - if [[ "${haveSourceUrls}" == true ]]; then - echo -ne " ${INFO} ${str}..." - fi + echo -ne " ${INFO} ${str}..." - # Parse into hosts file + # Parse into file gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}" # Format $parsedMatter line total as currency num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")") - if [[ "${haveSourceUrls}" == true ]]; then - echo -e "${OVER} ${TICK} ${str}" - fi - echo -e " ${INFO} Number of domains being pulled in by gravity: ${COL_BLUE}${num}${COL_NC}" + + echo -e "${OVER} ${TICK} ${str}" + echo -e " ${INFO} Gravity pulled in ${COL_BLUE}${num}${COL_NC} domains" str="Removing duplicate domains" - if [[ "${haveSourceUrls}" == true ]]; then - echo -ne " ${INFO} ${str}..." - fi - + echo -ne " ${INFO} ${str}..." sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}" chmod 644 "${piholeDir}/${preEventHorizon}" + echo -e "${OVER} ${TICK} ${str}" - if [[ "${haveSourceUrls}" == true ]]; then - echo -e "${OVER} ${TICK} ${str}" - # Format $preEventHorizon line total as currency - num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") - echo -e " ${INFO} Number of unique domains trapped in the Event Horizon: ${COL_BLUE}${num}${COL_NC}" - fi + # Format $preEventHorizon line total as currency + num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") + str="Storing ${COL_BLUE}${num}${COL_NC} unique blocking domains in database" + echo -ne " ${INFO} ${str}..." + database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}" + echo -e "${OVER} ${TICK} ${str}" } -# Whitelist user-defined domains -gravity_Whitelist() { - local num str - - if [[ ! -f "${whitelistFile}" ]]; then - echo -e " ${INFO} Nothing to whitelist!" - return 0 - fi - - num=$(wc -l < "${whitelistFile}") - str="Number of whitelisted domains: ${num}" - echo -ne " ${INFO} ${str}..." - - # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile - comm -23 "${piholeDir}/${preEventHorizon}" <(sort "${whitelistFile}") > "${piholeDir}/${whitelistMatter}" - chmod 644 "${piholeDir}/${whitelistMatter}" - - echo -e "${OVER} ${INFO} ${str}" +# Report number of entries in a table +gravity_Table_Count() { + local table="${1}" + local str="${2}" + local num + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table} WHERE enabled = 1;")" + echo -e " ${INFO} Number of ${str}: ${num}" } # Output count of blacklisted domains and regex filters -gravity_ShowBlockCount() { - local num - - if [[ -f "${blacklistFile}" ]]; then - num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")") - echo -e " ${INFO} Number of blacklisted domains: ${num}" - fi - - if [[ -f "${regexFile}" ]]; then - num=$(grep -cv "^#" "${regexFile}") - echo -e " ${INFO} Number of regex filters: ${num}" - fi +gravity_ShowCount() { + gravity_Table_Count "blacklist" "blacklisted domains" + gravity_Table_Count "whitelist" "whitelisted domains" + gravity_Table_Count "regex" "regex filters" } # Parse list of domains into hosts format @@ -553,7 +602,7 @@ gravity_ParseDomainsIntoHosts() { } # Create "localhost" entries into hosts format -gravity_ParseLocalDomains() { +gravity_generateLocalList() { local hostname if [[ -s "/etc/hostname" ]]; then @@ -579,43 +628,6 @@ gravity_ParseLocalDomains() { fi } -# Create primary blacklist entries -gravity_ParseBlacklistDomains() { - local output status - - # Empty $accretionDisc if it already exists, otherwise, create it - : > "${piholeDir}/${accretionDisc}" - - if [[ -f "${piholeDir}/${whitelistMatter}" ]]; then - mv "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}" - else - # There was no whitelist file, so use preEventHorizon instead of whitelistMatter. - cp -p "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}" - fi - chmod 644 "${piholeDir}/${accretionDisc}" - - # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it - output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 ) - status="$?" - - if [[ "${status}" -ne 0 ]]; then - echo -e "\\n ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n ${output}" - gravity_Cleanup "error" - fi - chmod 644 "${adList}" -} - -# Create user-added blacklist entries -gravity_ParseUserDomains() { - if [[ ! -f "${blacklistFile}" ]]; then - return 0 - fi - # Copy the file over as /etc/pihole/black.list so dnsmasq can use it - cp "${blacklistFile}" "${blackList}" 2> /dev/null || \ - echo -e "\\n ${CROSS} Unable to move ${blacklistFile##*/} to ${piholeDir}" - chmod 644 "${blackList}" -} - # Trap Ctrl-C gravity_Trap() { trap '{ echo -e "\\n\\n ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT @@ -647,6 +659,21 @@ gravity_Cleanup() { echo -e "${OVER} ${TICK} ${str}" + if ${optimize_database} ; then + str="Optimizing domains database" + echo -ne " ${INFO} ${str}..." + # Run VACUUM command on database to optimize it + output=$( { sqlite3 "${gravityDBfile}" "VACUUM;"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n ${output}" + error="error" + else + echo -e "${OVER} ${TICK} ${str}" + fi + fi + # Only restart DNS service if offline if ! pidof ${resolver} &> /dev/null; then "${PIHOLE_COMMAND}" restartdns @@ -673,17 +700,17 @@ Options: for var in "$@"; do case "${var}" in "-f" | "--force" ) forceDelete=true;; + "-o" | "--optimize" ) optimize_database=true;; "-h" | "--help" ) helpFunc;; - "-sd" | "--skip-download" ) skipDownload=true;; - "-b" | "--blacklist-only" ) listType="blacklist";; - "-w" | "--whitelist-only" ) listType="whitelist";; - "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";; esac done # Trap Ctrl-C gravity_Trap +# Move possibly existing legacy files to the gravity database +migrate_to_database + if [[ "${forceDelete:-}" == true ]]; then str="Deleting existing list cache" echo -ne "${INFO} ${str}..." @@ -692,56 +719,24 @@ if [[ "${forceDelete:-}" == true ]]; then echo -e "${OVER} ${TICK} ${str}" fi -detect_pihole_blocking_status - -# Determine which functions to run -if [[ "${skipDownload}" == false ]]; then - # Gravity needs to download blocklists - gravity_CheckDNSResolutionAvailable - gravity_GetBlocklistUrls - if [[ "${haveSourceUrls}" == true ]]; then - gravity_SetDownloadOptions - fi +# Gravity downloads blocklists next +gravity_CheckDNSResolutionAvailable +if gravity_GetBlocklistUrls; then + gravity_SetDownloadOptions + # Build preEventHorizon gravity_ConsolidateDownloadedBlocklists gravity_SortAndFilterConsolidatedList -else - # Gravity needs to modify Blacklist/Whitelist/Wildcards - echo -e " ${INFO} Using cached Event Horizon list..." - numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")") - echo -e " ${INFO} ${COL_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon" fi -# Perform when downloading blocklists, or modifying the whitelist -if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then - gravity_Whitelist -fi - -convert_wildcard_to_regex -gravity_ShowBlockCount - -# Perform when downloading blocklists, or modifying the white/blacklist (not wildcards) -if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then - str="Parsing domains into hosts format" - echo -ne " ${INFO} ${str}..." - - gravity_ParseUserDomains - - # Perform when downloading blocklists - if [[ ! "${listType:-}" == "blacklist" ]]; then - gravity_ParseLocalDomains - gravity_ParseBlacklistDomains - fi - - echo -e "${OVER} ${TICK} ${str}" - - gravity_Cleanup -fi +# Create local.list +gravity_generateLocalList +gravity_ShowCount +gravity_Cleanup echo "" # Determine if DNS has been restarted by this instance of gravity if [[ -z "${dnsWasOffline:-}" ]]; then - # Use "force-reload" when restarting dnsmasq for everything but Wildcards - "${PIHOLE_COMMAND}" restartdns "${dnsRestartType:-force-reload}" + "${PIHOLE_COMMAND}" restartdns reload fi "${PIHOLE_COMMAND}" status diff --git a/pihole b/pihole index 541434a0..71c286c6 100755 --- a/pihole +++ b/pihole @@ -10,10 +10,8 @@ # Please see LICENSE file for your rights under this license. readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" -readonly gravitylist="/etc/pihole/gravity.list" -readonly blacklist="/etc/pihole/black.list" -# setupVars is not readonly here because in some funcitons (checkout), +# setupVars is not readonly here because in some functions (checkout), # it might get set again when the installer is sourced. This causes an # error due to modifying a readonly variable. setupVars="/etc/pihole/setupVars.conf" @@ -148,16 +146,6 @@ Time: echo -e " ${INFO} Blocking already disabled, nothing to do" exit 0 fi - if [[ -e "${gravitylist}" ]]; then - mv "${gravitylist}" "${gravitylist}.bck" - echo "" > "${gravitylist}" - chmod 644 "${gravitylist}" - fi - if [[ -e "${blacklist}" ]]; then - mv "${blacklist}" "${blacklist}.bck" - echo "" > "${blacklist}" - chmod 644 "${blacklist}" - fi if [[ $# > 1 ]]; then local error=false if [[ "${2}" == *"s" ]]; then @@ -206,14 +194,6 @@ Time: echo -e " ${INFO} Enabling blocking" local str="Pi-hole Enabled" - if [[ -e "${gravitylist}.bck" ]]; then - mv "${gravitylist}.bck" "${gravitylist}" - chmod 644 "${gravitylist}" - fi - if [[ -e "${blacklist}.bck" ]]; then - mv "${blacklist}.bck" "${blacklist}" - chmod 644 "${blacklist}" - fi sed -i "/BLOCKING_ENABLED=/d" "${setupVars}" echo "BLOCKING_ENABLED=true" >> "${setupVars}" fi @@ -317,7 +297,7 @@ tailFunc() { # Colour everything else as gray tail -f /var/log/pihole.log | sed -E \ -e "s,($(date +'%b %d ')| dnsmasq[.*[0-9]]),,g" \ - -e "s,(.*(gravity.list|black.list|regex.list| config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \ + -e "s,(.*(gravity |black |regex | config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \ -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \ -e "s,.*,${COL_GRAY}&${COL_NC}," exit 0