mirror of
https://github.com/pi-hole/pi-hole.git
synced 2024-11-15 02:42:58 +00:00
Merge pull request #2832 from pi-hole/new/audit_database
Migrate audit list to gravity.db database table
This commit is contained in:
Mark Drobnak
GitHub
commit
445127accc
5 changed files with 114 additions and 34 deletions
35
advanced/Scripts/database_migration/gravity-db.sh
Normal file
35
advanced/Scripts/database_migration/gravity-db.sh
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
#!/usr/bin/env bash
|
||||
# shellcheck disable=SC1090
|
||||
|
||||
# Pi-hole: A black hole for Internet advertisements
|
||||
# (c) 2019 Pi-hole, LLC (https://pi-hole.net)
|
||||
# Network-wide ad blocking via your own hardware.
|
||||
#
|
||||
# Updates gravity.db database
|
||||
#
|
||||
# This file is copyright under the latest version of the EUPL.
|
||||
# Please see LICENSE file for your rights under this license.
|
||||
|
||||
upgrade_gravityDB(){
|
||||
local database piholeDir auditFile version
|
||||
database="${1}"
|
||||
piholeDir="${2}"
|
||||
auditFile="${piholeDir}/auditlog.list"
|
||||
|
||||
# Get database version
|
||||
version="$(sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")"
|
||||
|
||||
if [[ "$version" == "1" ]]; then
|
||||
# This migration script upgrades the gravity.db file by
|
||||
# adding the domain_audit table
|
||||
sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql"
|
||||
version=2
|
||||
|
||||
# Store audit domains in database table
|
||||
if [ -e "${auditFile}" ]; then
|
||||
echo -e " ${INFO} Migrating content of ${auditFile} into new database"
|
||||
# database_table_from_file is defined in gravity.sh
|
||||
database_table_from_file "domain_audit" "${auditFile}"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
10
advanced/Scripts/database_migration/gravity/1_to_2.sql
Normal file
10
advanced/Scripts/database_migration/gravity/1_to_2.sql
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
.timeout 30000
|
||||
|
||||
CREATE TABLE domain_audit
|
||||
(
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
domain TEXT UNIQUE NOT NULL,
|
||||
date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
|
||||
);
|
||||
|
||||
UPDATE info SET value = 2 WHERE property = 'version';
|
||||
|
|
@ -542,21 +542,45 @@ Teleporter() {
|
|||
php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-teleporter_${datetimestamp}.tar.gz"
|
||||
}
|
||||
|
||||
checkDomain()
|
||||
{
|
||||
local domain validDomain
|
||||
# Convert to lowercase
|
||||
domain="${1,,}"
|
||||
validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
|
||||
validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label
|
||||
echo "${validDomain}"
|
||||
}
|
||||
|
||||
addAudit()
|
||||
{
|
||||
shift # skip "-a"
|
||||
shift # skip "audit"
|
||||
for var in "$@"
|
||||
local domains validDomain
|
||||
domains=""
|
||||
for domain in "$@"
|
||||
do
|
||||
echo "${var}" >> /etc/pihole/auditlog.list
|
||||
# Check domain to be added. Only continue if it is valid
|
||||
validDomain="$(checkDomain "${domain}")"
|
||||
if [[ -n "${validDomain}" ]]; then
|
||||
# Put comma in between domains when there is
|
||||
# more than one domains to be added
|
||||
# SQL INSERT allows adding multiple rows at once using the format
|
||||
## INSERT INTO table (domain) VALUES ('abc.de'),('fgh.ij'),('klm.no'),('pqr.st');
|
||||
if [[ -n "${domains}" ]]; then
|
||||
domains="${domains},"
|
||||
fi
|
||||
domains="${domains}('${domain}')"
|
||||
fi
|
||||
done
|
||||
chmod 644 /etc/pihole/auditlog.list
|
||||
# Insert only the domain here. The date_added field will be
|
||||
# filled with its default value (date_added = current timestamp)
|
||||
sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};"
|
||||
}
|
||||
|
||||
clearAudit()
|
||||
{
|
||||
echo -n "" > /etc/pihole/auditlog.list
|
||||
chmod 644 /etc/pihole/auditlog.list
|
||||
sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;"
|
||||
}
|
||||
|
||||
SetPrivacyLevel() {
|
||||
|
|
|
|||
|
|
@ -80,6 +80,7 @@ CREATE TABLE gravity
|
|||
(
|
||||
domain TEXT PRIMARY KEY
|
||||
);
|
||||
|
||||
CREATE TABLE info
|
||||
(
|
||||
property TEXT PRIMARY KEY,
|
||||
|
|
|
|||
68
gravity.sh
68
gravity.sh
|
|
@ -17,6 +17,8 @@ coltable="/opt/pihole/COL_TABLE"
|
|||
source "${coltable}"
|
||||
regexconverter="/opt/pihole/wildcard_regex_converter.sh"
|
||||
source "${regexconverter}"
|
||||
# shellcheck disable=SC1091
|
||||
source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh"
|
||||
|
||||
basename="pihole"
|
||||
PIHOLE_COMMAND="/usr/local/bin/${basename}"
|
||||
|
|
@ -112,15 +114,21 @@ database_table_from_file() {
|
|||
inputfile="${source}"
|
||||
else
|
||||
# Apply format for white-, blacklist, regex, and adlist tables
|
||||
# Read file line by line
|
||||
local rowid
|
||||
declare -i rowid
|
||||
rowid=1
|
||||
# Read file line by line
|
||||
grep -v '^ *#' < "${source}" | while IFS= read -r domain
|
||||
do
|
||||
# Only add non-empty lines
|
||||
if [[ ! -z "${domain}" ]]; then
|
||||
echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
|
||||
if [[ -n "${domain}" ]]; then
|
||||
if [[ "${table}" == "domain_audit" ]]; then
|
||||
# domain_audit table format (no enable or modified fields)
|
||||
echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}"
|
||||
else
|
||||
# White-, black-, and regexlist format
|
||||
echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
|
||||
fi
|
||||
rowid+=1
|
||||
fi
|
||||
done
|
||||
|
|
@ -150,34 +158,36 @@ database_table_from_file() {
|
|||
# Migrate pre-v5.0 list files to database-based Pi-hole versions
|
||||
migrate_to_database() {
|
||||
# Create database file only if not present
|
||||
if [ -e "${gravityDBfile}" ]; then
|
||||
return 0
|
||||
if [ ! -e "${gravityDBfile}" ]; then
|
||||
# Create new database file - note that this will be created in version 1
|
||||
echo -e " ${INFO} Creating new gravity database"
|
||||
generate_gravity_database
|
||||
|
||||
# Migrate list files to new database
|
||||
if [ -e "${adListFile}" ]; then
|
||||
# Store adlist domains in database
|
||||
echo -e " ${INFO} Migrating content of ${adListFile} into new database"
|
||||
database_table_from_file "adlist" "${adListFile}"
|
||||
fi
|
||||
if [ -e "${blacklistFile}" ]; then
|
||||
# Store blacklisted domains in database
|
||||
echo -e " ${INFO} Migrating content of ${blacklistFile} into new database"
|
||||
database_table_from_file "blacklist" "${blacklistFile}"
|
||||
fi
|
||||
if [ -e "${whitelistFile}" ]; then
|
||||
# Store whitelisted domains in database
|
||||
echo -e " ${INFO} Migrating content of ${whitelistFile} into new database"
|
||||
database_table_from_file "whitelist" "${whitelistFile}"
|
||||
fi
|
||||
if [ -e "${regexFile}" ]; then
|
||||
# Store regex domains in database
|
||||
echo -e " ${INFO} Migrating content of ${regexFile} into new database"
|
||||
database_table_from_file "regex" "${regexFile}"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo -e " ${INFO} Creating new gravity database"
|
||||
generate_gravity_database
|
||||
|
||||
# Migrate list files to new database
|
||||
if [[ -e "${adListFile}" ]]; then
|
||||
# Store adlist domains in database
|
||||
echo -e " ${INFO} Migrating content of ${adListFile} into new database"
|
||||
database_table_from_file "adlist" "${adListFile}"
|
||||
fi
|
||||
if [[ -e "${blacklistFile}" ]]; then
|
||||
# Store blacklisted domains in database
|
||||
echo -e " ${INFO} Migrating content of ${blacklistFile} into new database"
|
||||
database_table_from_file "blacklist" "${blacklistFile}"
|
||||
fi
|
||||
if [[ -e "${whitelistFile}" ]]; then
|
||||
# Store whitelisted domains in database
|
||||
echo -e " ${INFO} Migrating content of ${whitelistFile} into new database"
|
||||
database_table_from_file "whitelist" "${whitelistFile}"
|
||||
fi
|
||||
if [[ -e "${regexFile}" ]]; then
|
||||
# Store regex domains in database
|
||||
echo -e " ${INFO} Migrating content of ${regexFile} into new database"
|
||||
database_table_from_file "regex" "${regexFile}"
|
||||
fi
|
||||
# Check if gravity database needs to be updated
|
||||
upgrade_gravityDB "${gravityDBfile}" "${piholeDir}"
|
||||
}
|
||||
|
||||
# Determine if DNS resolution is available before proceeding
|
||||
|
|
|
|||
Loading…
Reference in a new issue