Merge pull request #2611 from pi-hole/new/gravitydb

Store blocking domains in a database

advanced/01-pihole.conf

@@ -18,8 +18,6 @@
 #                    WITHIN /etc/dnsmasq.d/yourname.conf                      #
 ###############################################################################
 
-addn-hosts=/etc/pihole/gravity.list
-addn-hosts=/etc/pihole/black.list
 addn-hosts=/etc/pihole/local.list
 
 domain-needed

advanced/Scripts/list.sh

@@ -11,46 +11,45 @@
 # Globals
 basename=pihole
 piholeDir=/etc/"${basename}"
-whitelist="${piholeDir}"/whitelist.txt
+gravityDBfile="${piholeDir}/gravity.db"
-blacklist="${piholeDir}"/blacklist.txt
 
-readonly regexlist="/etc/pihole/regex.list"
 reload=false
 addmode=true
 verbose=true
 wildcard=false
+web=false
 
 domList=()
 
-listMain=""
+listType=""
-listAlt=""
+listname=""
 
 colfile="/opt/pihole/COL_TABLE"
 source ${colfile}
 
 
 helpFunc() {
-    if [[ "${listMain}" == "${whitelist}" ]]; then
+    if [[ "${listType}" == "whitelist" ]]; then
         param="w"
-        type="white"
+        type="whitelist"
-    elif [[ "${listMain}" == "${regexlist}" && "${wildcard}" == true ]]; then
+    elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then
         param="-wild"
-        type="wildcard black"
+        type="wildcard blacklist"
-    elif [[ "${listMain}" == "${regexlist}" ]]; then
+    elif [[ "${listType}" == "regex" ]]; then
         param="-regex"
-        type="regex black"
+        type="regex filter"
     else
         param="b"
-        type="black"
+        type="blacklist"
     fi
 
     echo "Usage: pihole -${param} [options] <domain> <domain2 ...>
 Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com'
-${type^}list one or more domains
+${type^} one or more domains
 
 Options:
-  -d, --delmode       Remove domain(s) from the ${type}list
+  -d, --delmode       Remove domain(s) from the ${type}
-  -nr, --noreload     Update ${type}list without refreshing dnsmasq
+  -nr, --noreload     Update ${type} without reloading the DNS server
   -q, --quiet         Make output less verbose
   -h, --help          Show this help dialog
   -l, --list          Display all your ${type}listed domains
@@ -73,7 +72,7 @@ HandleOther() {
 
     # Check validity of domain (don't check for regex entries)
     if [[ "${#domain}" -le 253 ]]; then
-        if [[ "${listMain}" == "${regexlist}" && "${wildcard}" == false ]]; then
+        if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then
             validDomain="${domain}"
         else
             validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
@@ -88,178 +87,143 @@ HandleOther() {
     fi
 }
 
-PoplistFile() {
+ProcessDomainList() {
-    # Check whitelist file exists, and if not, create it
+    if [[ "${listType}" == "regex" ]]; then
-    if [[ ! -f "${whitelist}" ]]; then
+        # Regex filter list
-        touch "${whitelist}"
+        listname="regex filters"
-        chmod 644 "${whitelist}"
+    else
-    fi
+        # Whitelist / Blacklist
-
+        listname="${listType}"
-    # Check blacklist file exists, and if not, create it
-    if [[ ! -f "${blacklist}" ]]; then
-        touch "${blacklist}"
-        chmod 644 "${blacklist}"
     fi
 
     for dom in "${domList[@]}"; do
-        # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other
+        # Format domain into regex filter if requested
+        if [[ "${wildcard}" == true ]]; then
+            dom="(^|\\.)${dom//\./\\.}$"
+        fi
+
+        # Logic: If addmode then add to desired list and remove from the other;
+        # if delmode then remove from desired list but do not add to the other
         if ${addmode}; then
-            AddDomain "${dom}" "${listMain}"
+            AddDomain "${dom}" "${listType}"
+            if [[ ! "${listType}" == "regex" ]]; then
                 RemoveDomain "${dom}" "${listAlt}"
+            fi
         else
-            RemoveDomain "${dom}" "${listMain}"
+            RemoveDomain "${dom}" "${listType}"
         fi
   done
 }
 
 AddDomain() {
+    local domain list num
+    # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
+    domain="$1"
     list="$2"
-    domain=$(EscapeRegexp "$1")
 
-    [[ "${list}" == "${whitelist}" ]] && listname="whitelist"
-    [[ "${list}" == "${blacklist}" ]] && listname="blacklist"
-
-    if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
-        [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
-        [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
-        bool=true
     # Is the domain in the list we want to add it to?
-        grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")"
 
-        if [[ "${bool}" == false ]]; then
+    if [[ "${num}" -ne 0 ]]; then
-            # Domain not found in the whitelist file, add it!
-            if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} Adding ${1} to ${listname}..."
-            fi
-            reload=true
-            # Add it to the list we want to add it to
-            echo "$1" >> "${list}"
-        else
       if [[ "${verbose}" == true ]]; then
           echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
       fi
+      return
     fi
-    elif [[ "${list}" == "${regexlist}" ]]; then
-        [[ -z "${type}" ]] && type="--wildcard-only"
-        bool=true
-        domain="${1}"
 
-        [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$"
+    # Domain not found in the table, add it!
-
-        # Is the domain in the list?
-        # Search only for exactly matching lines
-        grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
-
-        if [[ "${bool}" == false ]]; then
     if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} Adding ${domain} to regex list..."
+        echo -e "  ${INFO} Adding ${1} to the ${listname}..."
-            fi
-            reload="restart"
-            echo "$domain" >> "${regexlist}"
-        else
-            if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} ${domain} already exists in regex list, no need to add!"
-            fi
-        fi
     fi
+    reload=true
+    # Insert only the domain here. The enabled and date_added fields will be filled
+    # with their default values (enabled = true, date_added = current timestamp)
+    sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES ('${domain}');"
 }
 
 RemoveDomain() {
+    local domain list num
+    # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
+    domain="$1"
     list="$2"
-    domain=$(EscapeRegexp "$1")
 
-    [[ "${list}" == "${whitelist}" ]] && listname="whitelist"
+    # Is the domain in the list we want to remove it from?
-    [[ "${list}" == "${blacklist}" ]] && listname="blacklist"
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")"
 
-    if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
+    if [[ "${num}" -eq 0 ]]; then
-        bool=true
-        [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
-        [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
-        # Is it in the list? Logic follows that if its whitelisted it should not be blacklisted and vice versa
-        grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
-        if [[ "${bool}" == true ]]; then
-            # Remove it from the other one
-            echo -e "  ${INFO} Removing $1 from ${listname}..."
-            # /I flag: search case-insensitive
-            sed -i "/${domain}/Id" "${list}"
-            reload=true
-        else
       if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} ${1} does not exist in ${listname}, no need to remove!"
+          echo -e "  ${INFO} ${1} does not exist in ${list}, no need to remove!"
       fi
+      return
     fi
-    elif [[ "${list}" == "${regexlist}" ]]; then
-        [[ -z "${type}" ]] && type="--wildcard-only"
-        domain="${1}"
 
-        [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$"
+    # Domain found in the table, remove it!
-
-        bool=true
-        # Is it in the list?
-        grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
-        if [[ "${bool}" == true ]]; then
-            # Remove it from the other one
-            echo -e "  ${INFO} Removing $domain from regex list..."
-            local lineNumber
-            lineNumber=$(grep -Fnx "$domain" "${list}" | cut -f1 -d:)
-            sed -i "${lineNumber}d" "${list}"
-            reload=true
-        else
     if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} ${domain} does not exist in regex list, no need to remove!"
+        echo -e "  ${INFO} Removing ${1} from the ${listname}..."
     fi
-        fi
+    reload=true
-    fi
+    # Remove it from the current list
-}
+    sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = '${domain}';"
-
-# Update Gravity
-Reload() {
-    echo ""
-    pihole -g --skip-download "${type:-}"
 }
 
 Displaylist() {
-    if [[ -f ${listMain} ]]; then
+    local list listname count num_pipes domain enabled status nicedate
-        if [[ "${listMain}" == "${whitelist}" ]]; then
+
-            string="gravity resistant domains"
+    listname="${listType}"
+    data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)"
+
+    if [[ -z $data ]]; then
+        echo -e "Not showing empty ${listname}"
     else
-            string="domains caught in the sinkhole"
+        echo -e "Displaying ${listname}:"
-        fi
-        verbose=false
-        echo -e "Displaying $string:\n"
         count=1
-        while IFS= read -r RD || [ -n "${RD}" ]; do
+        while IFS= read -r line
-            echo "  ${count}: ${RD}"
+        do
-            count=$((count+1))
+            # Count number of pipes seen in this line
-        done < "${listMain}"
+            # This is necessary because we can only detect the pipe separating the fields
+            # from the end backwards as the domain (which is the first field) may contain
+            # pipe symbols as they are perfectly valid regex filter control characters
+            num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")"
+
+            # Extract domain and enabled status based on the obtained number of pipe characters
+            domain="$(cut -d'|' -f"-$((num_pipes-1))" <<< "${line}")"
+            enabled="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")"
+            datemod="$(cut -d'|' -f"$((num_pipes+1))" <<< "${line}")"
+
+            # Translate boolean status into human readable string
+            if [[ "${enabled}" -eq 1 ]]; then
+                status="enabled"
             else
-        echo -e "  ${COL_LIGHT_RED}${listMain} does not exist!${COL_NC}"
+                status="disabled"
+            fi
+
+            # Get nice representation of numerical date stored in database
+            nicedate=$(date --rfc-2822 -d "@${datemod}")
+
+            echo "  ${count}: ${domain} (${status}, last modified ${nicedate})"
+            count=$((count+1))
+        done <<< "${data}"
     fi
     exit 0;
 }
 
 NukeList() {
-    if [[ -f "${listMain}" ]]; then
+    sqlite3 "${gravityDBfile}" "DELETE FROM ${listType};"
-        # Back up original list
-        cp -p "${listMain}" "${listMain}.bck~"
-        # Empty out file
-        echo "" > "${listMain}"
-        chmod 644 "${listMain}"
-    fi
 }
 
 for var in "$@"; do
     case "${var}" in
-        "-w" | "whitelist"   ) listMain="${whitelist}"; listAlt="${blacklist}";;
+        "-w" | "whitelist"   ) listType="whitelist"; listAlt="blacklist";;
-        "-b" | "blacklist"   ) listMain="${blacklist}"; listAlt="${whitelist}";;
+        "-b" | "blacklist"   ) listType="blacklist"; listAlt="whitelist";;
-        "--wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;;
+        "--wild" | "wildcard" ) listType="regex"; wildcard=true;;
-        "--regex" | "regex"   ) listMain="${regexlist}";;
+        "--regex" | "regex"   ) listType="regex";;
         "-nr"| "--noreload"  ) reload=false;;
         "-d" | "--delmode"   ) addmode=false;;
         "-q" | "--quiet"     ) verbose=false;;
         "-h" | "--help"      ) helpFunc;;
         "-l" | "--list"      ) Displaylist;;
         "--nuke"             ) NukeList;;
+        "--web"              ) web=true;;
         *                    ) HandleOther "${var}";;
     esac
 done
@@ -270,9 +234,13 @@ if [[ $# = 0 ]]; then
     helpFunc
 fi
 
-PoplistFile
+ProcessDomainList
+
+# Used on web interface
+if $web; then
+echo "DONE"
+fi
 
 if [[ "${reload}" != false ]]; then
-    # Ensure that "restart" is used for Wildcard updates
+    pihole restartdns reload
-    Reload "${reload}"
 fi

advanced/Scripts/query.sh

@@ -11,7 +11,7 @@
 
 # Globals
 piholeDir="/etc/pihole"
-adListsList="$piholeDir/adlists.list"
+gravityDBfile="${piholeDir}/gravity.db"
 wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf"
 options="$*"
 adlist=""
@@ -73,11 +73,6 @@ Options:
   exit 0
 fi
 
-if [[ ! -e "$adListsList" ]]; then
-    echo -e "${COL_LIGHT_RED}The file $adListsList was not found${COL_NC}"
-    exit 1
-fi
-
 # Handle valid options
 if [[ "${options}" == *"-bp"* ]]; then
     exact="exact"; blockpage=true
@@ -186,11 +181,8 @@ fi
 
 # Get adlist file content as array
 if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
-    for adlistUrl in $(< "${adListsList}"); do
+    # Retrieve source URLs from gravity database
-        if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then
+    mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)"
-            adlists+=("${adlistUrl}")
-        fi
-    done
 fi
 
 # Print "Exact matches for" title

advanced/Scripts/webpage.sh

@@ -17,6 +17,8 @@ readonly FTLconf="/etc/pihole/pihole-FTL.conf"
 # 03 -> wildcards
 readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf"
 
+readonly gravityDBfile="/etc/pihole/gravity.db"
+
 coltable="/opt/pihole/COL_TABLE"
 if [[ -f ${coltable} ]]; then
     source ${coltable}
@@ -386,19 +388,17 @@ SetWebUILayout() {
 }
 
 CustomizeAdLists() {
-    list="/etc/pihole/adlists.list"
+    local address
+    address="${args[3]}"
 
     if [[ "${args[2]}" == "enable" ]]; then
-        sed -i "\\@${args[3]}@s/^#http/http/g" "${list}"
+        sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 1 WHERE address = '${address}'"
     elif [[ "${args[2]}" == "disable" ]]; then
-        sed -i "\\@${args[3]}@s/^http/#http/g" "${list}"
+        sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 0 WHERE address = '${address}'"
     elif [[ "${args[2]}" == "add" ]]; then
-        if [[ $(grep -c "^${args[3]}$" "${list}") -eq 0 ]] ; then
+        sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlists (address) VALUES ('${address}')"
-            echo "${args[3]}" >> ${list}
-        fi
     elif [[ "${args[2]}" == "del" ]]; then
-        var=$(echo "${args[3]}" | sed 's/\//\\\//g')
+        sqlite3 "${gravityDBfile}" "DELETE FROM adlists WHERE address = '${address}'"
-        sed -i "/${var}/Id" "${list}"
     else
         echo "Not permitted"
         return 1

advanced/Templates/gravity.db.sql

@@ -0,0 +1,92 @@
+CREATE TABLE whitelist
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	domain TEXT UNIQUE NOT NULL,
+	enabled BOOLEAN NOT NULL DEFAULT 1,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+CREATE TABLE blacklist
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	domain TEXT UNIQUE NOT NULL,
+	enabled BOOLEAN NOT NULL DEFAULT 1,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+CREATE TABLE regex
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	domain TEXT UNIQUE NOT NULL,
+	enabled BOOLEAN NOT NULL DEFAULT 1,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+CREATE TABLE adlists
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	address TEXT UNIQUE NOT NULL,
+	enabled BOOLEAN NOT NULL DEFAULT 1,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+CREATE TABLE gravity
+(
+	domain TEXT PRIMARY KEY
+);
+CREATE TABLE info
+(
+	property TEXT PRIMARY KEY,
+	value TEXT NOT NULL
+);
+
+INSERT INTO info VALUES("version","1");
+
+CREATE VIEW vw_gravity AS SELECT a.domain
+    FROM gravity a
+    WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1);
+
+CREATE VIEW vw_whitelist AS SELECT a.domain
+    FROM whitelist a
+    WHERE a.enabled == 1
+    ORDER BY a.id;
+
+CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist
+    BEGIN
+      UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+    END;
+
+CREATE VIEW vw_blacklist AS SELECT a.domain
+    FROM blacklist a
+    WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist
+    ORDER BY a.id;
+
+CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist
+    BEGIN
+      UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+    END;
+
+CREATE VIEW vw_regex AS SELECT a.domain
+    FROM regex a
+    WHERE a.enabled == 1
+    ORDER BY a.id;
+
+CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex
+    BEGIN
+      UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+    END;
+
+CREATE VIEW vw_adlists AS SELECT a.address
+    FROM adlists a
+    WHERE a.enabled == 1
+    ORDER BY a.id;
+
+CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists
+    BEGIN
+      UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address;
+    END;
+

automated install/basic-install.sh

@@ -1234,6 +1234,7 @@ version_check_dnsmasq() {
     local dnsmasq_conf="/etc/dnsmasq.conf"
     local dnsmasq_conf_orig="/etc/dnsmasq.conf.orig"
     local dnsmasq_pihole_id_string="addn-hosts=/etc/pihole/gravity.list"
+    local dnsmasq_pihole_id_string2="# Dnsmasq config for Pi-hole's FTLDNS"
     local dnsmasq_original_config="${PI_HOLE_LOCAL_REPO}/advanced/dnsmasq.conf.original"
     local dnsmasq_pihole_01_snippet="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf"
     local dnsmasq_pihole_01_location="/etc/dnsmasq.d/01-pihole.conf"
@@ -1241,8 +1242,9 @@ version_check_dnsmasq() {
     # If the dnsmasq config file exists
     if [[ -f "${dnsmasq_conf}" ]]; then
         printf "  %b Existing dnsmasq.conf found..." "${INFO}"
-        # If gravity.list is found within this file, we presume it's from older versions on Pi-hole,
+        # If a specific string is found within this file, we presume it's from older versions on Pi-hole,
-        if grep -q ${dnsmasq_pihole_id_string} ${dnsmasq_conf}; then
+        if grep -q "${dnsmasq_pihole_id_string}" "${dnsmasq_conf}" ||
+           grep -q "${dnsmasq_pihole_id_string2}" "${dnsmasq_conf}"; then
             printf " it is from a previous Pi-hole install.\\n"
             printf "  %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}"
             # so backup the original file

gravity.sh

@@ -23,31 +23,27 @@ PIHOLE_COMMAND="/usr/local/bin/${basename}"
 
 piholeDir="/etc/${basename}"
 
-adListFile="${piholeDir}/adlists.list"
+# Legacy (pre v5.0) list file locations
-adListDefault="${piholeDir}/adlists.default"
-
 whitelistFile="${piholeDir}/whitelist.txt"
 blacklistFile="${piholeDir}/blacklist.txt"
 regexFile="${piholeDir}/regex.list"
+adListFile="${piholeDir}/adlists.list"
 
-adList="${piholeDir}/gravity.list"
-blackList="${piholeDir}/black.list"
 localList="${piholeDir}/local.list"
 VPNList="/etc/openvpn/ipp.txt"
 
+piholeGitDir="/etc/.pihole"
+gravityDBfile="${piholeDir}/gravity.db"
+gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql"
+optimize_database=false
+
 domainsExtension="domains"
 matterAndLight="${basename}.0.matterandlight.txt"
 parsedMatter="${basename}.1.parsedmatter.txt"
-whitelistMatter="${basename}.2.whitelistmatter.txt"
-accretionDisc="${basename}.3.accretionDisc.txt"
 preEventHorizon="list.preEventHorizon"
 
-skipDownload="false"
-
 resolver="pihole-FTL"
 
-haveSourceUrls=true
-
 # Source setupVars from install script
 setupVars="${piholeDir}/setupVars.conf"
 if [[ -f "${setupVars}" ]];then
@@ -83,17 +79,104 @@ if [[ -r "${piholeDir}/pihole.conf" ]]; then
   echo -e "  ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}"
 fi
 
-# Determine if Pi-hole blocking is disabled
+# Generate new sqlite3 file from schema template
-# If this is the case, we want to update
+generate_gravity_database() {
-#  gravity.list.bck and black.list.bck instead of
+  sqlite3 "${gravityDBfile}" < "${gravityDBschema}"
-#  gravity.list and black.list
+}
-detect_pihole_blocking_status() {
+
-  if [[ "${BLOCKING_ENABLED}" == false ]]; then
+# Import domains from file and store them in the specified database table
-    echo -e "  ${INFO} Pi-hole blocking is disabled"
+database_table_from_file() {
-    adList="${adList}.bck"
+  # Define locals
-    blackList="${blackList}.bck"
+  local table source backup_path backup_file
+  table="${1}"
+  source="${2}"
+  backup_path="${piholeDir}/migration_backup"
+  backup_file="${backup_path}/$(basename "${2}")"
+
+  # Truncate table
+  output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to truncate ${table} database ${gravityDBfile}\\n  ${output}"
+    gravity_Cleanup "error"
+  fi
+
+  local tmpFile
+  tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")"
+  local timestamp
+  timestamp="$(date --utc +'%s')"
+  local inputfile
+  if [[ "${table}" == "gravity" ]]; then
+    # No need to modify the input data for the gravity table
+    inputfile="${source}"
   else
-    echo -e "  ${INFO} Pi-hole blocking is enabled"
+    # Apply format for white-, blacklist, regex, and adlists tables
+    local rowid
+    declare -i rowid
+    rowid=1
+    # Read file line by line
+    grep -v '^ *#' < "${source}" | while IFS= read -r domain
+    do
+      # Only add non-empty lines
+      if [[ ! -z "${domain}" ]]; then
+        echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
+        rowid+=1
+      fi
+    done
+    inputfile="${tmpFile}"
+  fi
+  # Store domains in database table specified by ${table}
+  # Use printf as .mode and .import need to be on separate lines
+  # see https://unix.stackexchange.com/a/445615/83260
+  output=$( { printf ".mode csv\\n.import \"%s\" %s\\n" "${inputfile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to fill table ${table} in database ${gravityDBfile}\\n  ${output}"
+    gravity_Cleanup "error"
+  fi
+
+  # Delete tmpfile
+  rm "${tmpFile}" > /dev/null 2>&1 || \
+      echo -e "  ${CROSS} Unable to remove ${tmpFile}"
+
+  # Move source file to backup directory, create directory if not existing
+  mkdir -p "${backup_path}"
+  mv "${source}" "${backup_file}" 2> /dev/null || \
+      echo -e "  ${CROSS} Unable to backup ${source} to ${backup_path}"
+}
+
+# Migrate pre-v5.0 list files to database-based Pi-hole versions
+migrate_to_database() {
+  # Create database file only if not present
+  if [ -e "${gravityDBfile}" ]; then
+    return 0
+  fi
+
+  echo -e "  ${INFO} Creating new gravity database"
+  generate_gravity_database
+
+  # Migrate list files to new database
+  if [[ -e "${adListFile}" ]]; then
+    # Store adlists domains in database
+    echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
+    database_table_from_file "adlists" "${adListFile}"
+  fi
+  if [[ -e "${blacklistFile}" ]]; then
+    # Store blacklisted domains in database
+    echo -e "  ${INFO} Migrating content of ${blacklistFile} into new database"
+    database_table_from_file "blacklist" "${blacklistFile}"
+  fi
+  if [[ -e "${whitelistFile}" ]]; then
+    # Store whitelisted domains in database
+    echo -e "  ${INFO} Migrating content of ${whitelistFile} into new database"
+    database_table_from_file "whitelist" "${whitelistFile}"
+  fi
+  if [[ -e "${regexFile}" ]]; then
+    # Store regex domains in database
+    echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
+    database_table_from_file "regex" "${regexFile}"
   fi
 }
 
@@ -157,15 +240,9 @@ gravity_CheckDNSResolutionAvailable() {
 gravity_GetBlocklistUrls() {
   echo -e "  ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
 
-  if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then
+  # Retrieve source URLs from gravity database
-    # Remove superceded $adListDefault file
+  # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true)
-    rm "${adListDefault}" 2> /dev/null || \
+  mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)"
-      echo -e "  ${CROSS} Unable to remove ${adListDefault}"
-  fi
-
-  # Retrieve source URLs from $adListFile
-  # Logic: Remove comments and empty lines
-  mapfile -t sources <<< "$(grep -v -E "^(#|$)" "${adListFile}" 2> /dev/null)"
 
   # Parse source domains from $sources
   mapfile -t sourceDomains <<< "$(
@@ -182,11 +259,12 @@ gravity_GetBlocklistUrls() {
 
   if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then
     echo -e "${OVER}  ${TICK} ${str}"
+    return 0
   else
     echo -e "${OVER}  ${CROSS} ${str}"
     echo -e "  ${INFO} No source list found, or it is empty"
     echo ""
-    haveSourceUrls=false
+    return 1
   fi
 }
 
@@ -214,11 +292,9 @@ gravity_SetDownloadOptions() {
       *) cmd_ext="";;
     esac
 
-    if [[ "${skipDownload}" == false ]]; then
     echo -e "  ${INFO} Target: ${domain} (${url##*/})"
     gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}"
     echo ""
-    fi
   done
   gravity_Blackbody=true
 }
@@ -439,9 +515,7 @@ gravity_ConsolidateDownloadedBlocklists() {
   local str lastLine
 
   str="Consolidating blocklists"
-  if [[ "${haveSourceUrls}" == true ]]; then
   echo -ne "  ${INFO} ${str}..."
-  fi
 
   # Empty $matterAndLight if it already exists, otherwise, create it
   : > "${piholeDir}/${matterAndLight}"
@@ -461,9 +535,8 @@ gravity_ConsolidateDownloadedBlocklists() {
       fi
     fi
   done
-  if [[ "${haveSourceUrls}" == true ]]; then
   echo -e "${OVER}  ${TICK} ${str}"
-  fi
+
 }
 
 # Parse consolidated list into (filtered, unique) domains-only format
@@ -471,69 +544,45 @@ gravity_SortAndFilterConsolidatedList() {
   local str num
 
   str="Extracting domains from blocklists"
-  if [[ "${haveSourceUrls}" == true ]]; then
   echo -ne "  ${INFO} ${str}..."
-  fi
 
-  # Parse into hosts file
+  # Parse into file
   gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}"
 
   # Format $parsedMatter line total as currency
   num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")")
-  if [[ "${haveSourceUrls}" == true ]]; then
+
   echo -e "${OVER}  ${TICK} ${str}"
-  fi
+  echo -e "  ${INFO} Gravity pulled in ${COL_BLUE}${num}${COL_NC} domains"
-  echo -e "  ${INFO} Number of domains being pulled in by gravity: ${COL_BLUE}${num}${COL_NC}"
 
   str="Removing duplicate domains"
-  if [[ "${haveSourceUrls}" == true ]]; then
   echo -ne "  ${INFO} ${str}..."
-  fi
-
   sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}"
   chmod 644 "${piholeDir}/${preEventHorizon}"
-
-  if [[ "${haveSourceUrls}" == true ]]; then
   echo -e "${OVER}  ${TICK} ${str}"
+
   # Format $preEventHorizon line total as currency
   num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
-    echo -e "  ${INFO} Number of unique domains trapped in the Event Horizon: ${COL_BLUE}${num}${COL_NC}"
+  str="Storing ${COL_BLUE}${num}${COL_NC} unique blocking domains in database"
-  fi
+  echo -ne "  ${INFO} ${str}..."
+  database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}"
+  echo -e "${OVER}  ${TICK} ${str}"
 }
 
-# Whitelist user-defined domains
+# Report number of entries in a table
-gravity_Whitelist() {
+gravity_Table_Count() {
-  local num str
+  local table="${1}"
-
+  local str="${2}"
-  if [[ ! -f "${whitelistFile}" ]]; then
+  local num
-    echo -e "  ${INFO} Nothing to whitelist!"
+  num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table} WHERE enabled = 1;")"
-    return 0
+  echo -e "  ${INFO} Number of ${str}: ${num}"
-  fi
-
-  num=$(wc -l < "${whitelistFile}")
-  str="Number of whitelisted domains: ${num}"
-  echo -ne "  ${INFO} ${str}..."
-
-  # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile
-  comm -23 "${piholeDir}/${preEventHorizon}" <(sort "${whitelistFile}") > "${piholeDir}/${whitelistMatter}"
-  chmod 644 "${piholeDir}/${whitelistMatter}"
-
-  echo -e "${OVER}  ${INFO} ${str}"
 }
 
 # Output count of blacklisted domains and regex filters
-gravity_ShowBlockCount() {
+gravity_ShowCount() {
-  local num
+  gravity_Table_Count "blacklist" "blacklisted domains"
-
+  gravity_Table_Count "whitelist" "whitelisted domains"
-  if [[ -f "${blacklistFile}" ]]; then
+  gravity_Table_Count "regex" "regex filters"
-    num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
-    echo -e "  ${INFO} Number of blacklisted domains: ${num}"
-  fi
-
-  if [[ -f "${regexFile}" ]]; then
-    num=$(grep -cv "^#" "${regexFile}")
-    echo -e "  ${INFO} Number of regex filters: ${num}"
-  fi
 }
 
 # Parse list of domains into hosts format
@@ -553,7 +602,7 @@ gravity_ParseDomainsIntoHosts() {
 }
 
 # Create "localhost" entries into hosts format
-gravity_ParseLocalDomains() {
+gravity_generateLocalList() {
   local hostname
 
   if [[ -s "/etc/hostname" ]]; then
@@ -579,43 +628,6 @@ gravity_ParseLocalDomains() {
   fi
 }
 
-# Create primary blacklist entries
-gravity_ParseBlacklistDomains() {
-  local output status
-
-  # Empty $accretionDisc if it already exists, otherwise, create it
-  : > "${piholeDir}/${accretionDisc}"
-
-  if [[ -f "${piholeDir}/${whitelistMatter}" ]]; then
-    mv "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}"
-  else
-    # There was no whitelist file, so use preEventHorizon instead of whitelistMatter.
-    cp -p "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}"
-  fi
-  chmod 644 "${piholeDir}/${accretionDisc}"
-
-  # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it
-  output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 )
-  status="$?"
-
-  if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n  ${output}"
-    gravity_Cleanup "error"
-  fi
-  chmod 644 "${adList}"
-}
-
-# Create user-added blacklist entries
-gravity_ParseUserDomains() {
-  if [[ ! -f "${blacklistFile}" ]]; then
-    return 0
-  fi
-  # Copy the file over as /etc/pihole/black.list so dnsmasq can use it
-  cp "${blacklistFile}" "${blackList}" 2> /dev/null || \
-    echo -e "\\n  ${CROSS} Unable to move ${blacklistFile##*/} to ${piholeDir}"
-  chmod 644 "${blackList}"
-}
-
 # Trap Ctrl-C
 gravity_Trap() {
   trap '{ echo -e "\\n\\n  ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
@@ -647,6 +659,21 @@ gravity_Cleanup() {
 
   echo -e "${OVER}  ${TICK} ${str}"
 
+  if ${optimize_database} ; then
+    str="Optimizing domains database"
+    echo -ne "  ${INFO} ${str}..."
+    # Run VACUUM command on database to optimize it
+    output=$( { sqlite3 "${gravityDBfile}" "VACUUM;"; } 2>&1 )
+    status="$?"
+
+    if [[ "${status}" -ne 0 ]]; then
+      echo -e "\\n  ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n  ${output}"
+      error="error"
+    else
+      echo -e "${OVER}  ${TICK} ${str}"
+    fi
+  fi
+
   # Only restart DNS service if offline
   if ! pidof ${resolver} &> /dev/null; then
     "${PIHOLE_COMMAND}" restartdns
@@ -673,17 +700,17 @@ Options:
 for var in "$@"; do
   case "${var}" in
     "-f" | "--force" ) forceDelete=true;;
+    "-o" | "--optimize" ) optimize_database=true;;
     "-h" | "--help" ) helpFunc;;
-    "-sd" | "--skip-download" ) skipDownload=true;;
-    "-b" | "--blacklist-only" ) listType="blacklist";;
-    "-w" | "--whitelist-only" ) listType="whitelist";;
-    "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";;
   esac
 done
 
 # Trap Ctrl-C
 gravity_Trap
 
+# Move possibly existing legacy files to the gravity database
+migrate_to_database
+
 if [[ "${forceDelete:-}" == true ]]; then
   str="Deleting existing list cache"
   echo -ne "${INFO} ${str}..."
@@ -692,56 +719,24 @@ if [[ "${forceDelete:-}" == true ]]; then
   echo -e "${OVER}  ${TICK} ${str}"
 fi
 
-detect_pihole_blocking_status
+# Gravity downloads blocklists next
-
-# Determine which functions to run
-if [[ "${skipDownload}" == false ]]; then
-  # Gravity needs to download blocklists
 gravity_CheckDNSResolutionAvailable
-  gravity_GetBlocklistUrls
+if gravity_GetBlocklistUrls; then
-  if [[ "${haveSourceUrls}" == true ]]; then
   gravity_SetDownloadOptions
-  fi
+  # Build preEventHorizon
   gravity_ConsolidateDownloadedBlocklists
   gravity_SortAndFilterConsolidatedList
-else
-  # Gravity needs to modify Blacklist/Whitelist/Wildcards
-  echo -e "  ${INFO} Using cached Event Horizon list..."
-  numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
-  echo -e "  ${INFO} ${COL_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon"
 fi
 
-# Perform when downloading blocklists, or modifying the whitelist
+# Create local.list
-if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then
+gravity_generateLocalList
-  gravity_Whitelist
+gravity_ShowCount
-fi
-
-convert_wildcard_to_regex
-gravity_ShowBlockCount
-
-# Perform when downloading blocklists, or modifying the white/blacklist (not wildcards)
-if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then
-  str="Parsing domains into hosts format"
-  echo -ne "  ${INFO} ${str}..."
-
-  gravity_ParseUserDomains
-
-  # Perform when downloading blocklists
-  if [[ ! "${listType:-}" == "blacklist" ]]; then
-    gravity_ParseLocalDomains
-    gravity_ParseBlacklistDomains
-  fi
-
-  echo -e "${OVER}  ${TICK} ${str}"
 
 gravity_Cleanup
-fi
-
 echo ""
 
 # Determine if DNS has been restarted by this instance of gravity
 if [[ -z "${dnsWasOffline:-}" ]]; then
-  # Use "force-reload" when restarting dnsmasq for everything but Wildcards
+  "${PIHOLE_COMMAND}" restartdns reload
-  "${PIHOLE_COMMAND}" restartdns "${dnsRestartType:-force-reload}"
 fi
 "${PIHOLE_COMMAND}" status

pihole

@@ -10,10 +10,8 @@
 # Please see LICENSE file for your rights under this license.
 
 readonly PI_HOLE_SCRIPT_DIR="/opt/pihole"
-readonly gravitylist="/etc/pihole/gravity.list"
-readonly blacklist="/etc/pihole/black.list"
 
-# setupVars is not readonly here because in some funcitons (checkout),
+# setupVars is not readonly here because in some functions (checkout),
 # it might get set again when the installer is sourced. This causes an
 # error due to modifying a readonly variable.
 setupVars="/etc/pihole/setupVars.conf"
@@ -148,16 +146,6 @@ Time:
       echo -e "  ${INFO} Blocking already disabled, nothing to do"
       exit 0
     fi
-    if [[ -e "${gravitylist}" ]]; then
-      mv "${gravitylist}" "${gravitylist}.bck"
-      echo "" > "${gravitylist}"
-	  chmod 644 "${gravitylist}"
-    fi
-    if [[ -e "${blacklist}" ]]; then
-      mv "${blacklist}" "${blacklist}.bck"
-      echo "" > "${blacklist}"
-	  chmod 644 "${blacklist}"
-    fi
     if [[ $# > 1 ]]; then
       local error=false
       if [[ "${2}" == *"s" ]]; then
@@ -206,14 +194,6 @@ Time:
     echo -e "  ${INFO} Enabling blocking"
     local str="Pi-hole Enabled"
 
-    if [[ -e "${gravitylist}.bck" ]]; then
-      mv "${gravitylist}.bck" "${gravitylist}"
-	  chmod 644 "${gravitylist}"
-    fi
-    if [[ -e "${blacklist}.bck" ]]; then
-      mv "${blacklist}.bck" "${blacklist}"
-	  chmod 644 "${blacklist}"
-    fi
     sed -i "/BLOCKING_ENABLED=/d" "${setupVars}"
     echo "BLOCKING_ENABLED=true" >> "${setupVars}"
   fi
@@ -317,7 +297,7 @@ tailFunc() {
   # Colour everything else as gray
   tail -f /var/log/pihole.log | sed -E \
     -e "s,($(date +'%b %d ')| dnsmasq[.*[0-9]]),,g" \
-    -e "s,(.*(gravity.list|black.list|regex.list| config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \
+    -e "s,(.*(gravity |black |regex | config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \
     -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \
     -e "s,.*,${COL_GRAY}&${COL_NC},"
   exit 0